def getTimeOfLastUpdateInLogs(self):
"""
@summary : Returns the time of the last update in iso format.
@return : None if no update as found, EPCH is returned in iso format,
as to make sure an update is made since no prior updates exist.
"""
timeOfLastUpdate = StatsDateLib.getIsoTodaysMidnight( StatsDateLib.getCurrentTimeInIsoformat() )
paths = StatsPaths()
paths.setPaths()
updatesDirectory = paths.STATSTEMPAUTUPDTLOGS + self.updateType + "/"
if not os.path.isdir( updatesDirectory ):
os.makedirs(updatesDirectory)
allEntries = os.listdir(updatesDirectory)
if allEntries !=[] :
allEntries.sort()
allEntries.reverse()
timeOfLastUpdate = os.path.basename( allEntries[0] ).replace( "_"," " )
return timeOfLastUpdate
def setMonths( self ):
"""
Returns the 3 months including current month.
"""
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentDate = datetime.date( int(currentTime[0:4]), int(currentTime[5:7]), 1 )
months = []
for i in range(0,5):
if currentDate.month -i < 1 :
month = currentDate.month -i + 12
year = currentDate.year -i
else :
month = currentDate.month -i
year = currentDate.year
newdate = StatsDateLib.getSecondsSinceEpoch( "%s-%s-01 00:00:00" %( year,month ) )
months.append( newdate )
#print year,month,day
months.reverse()
self.months = months
def setMonths( self ):
"""
@Summary : Sets the months value to an array containing
the last X months in "since epoch" numbers
based on the globally set NB_MONTHS_DISPLAYED
value.
"""
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentDate = datetime.date( int(currentTime[0:4]), int(currentTime[5:7]), 1 ) # day always = 1 in case currentDate.day > 28
months = []
for i in range(0,NB_MONTHS_DISPLAYED):
if currentDate.month - (i%12) < 1 :
month = currentDate.month - (i%12)+12
if currentDate.month -i < 1:
year = currentDate.year - int( abs(math.floor( float( ( currentDate.month - i ) / 12 ) ) ) )
else :
month = currentDate.month - i
year = currentDate.year
months.append( StatsDateLib.getSecondsSinceEpoch( "%s-%s-%s 00:00:00" %(year,month,"01") ) )
months.reverse()
self.months = months
print months
def main():
"""
@summary : This program is to be used to backup
rrd databases and their corresponding
time of update files. Backing up rrd
databases at various point in time is a
recommended paractice in case newly
entered data is not valid.
"""
setGlobalLanguageParameters()
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentTime = StatsDateLib.getIsoWithRoundedSeconds( currentTime )
currentTime = currentTime.replace(" ", "_")
backupsToKeep = 20
if len( sys.argv ) == 2:
try:
backupsToKeep = int( sys.argv[1] )
except:
print _( "Days to keep value must be an integer. For default 20 backups value, type nothing." )
sys.exit()
backupDatabaseUpdateTimes( currentTime, backupsToKeep )
backupDatabases( currentTime, backupsToKeep )
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : start, end tuple in iso format.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
currentDate = datetime.date( int(currentTime[0:4]), int(currentTime[5:7]), 1 )
nbMonthsToRevwind = NB_MONTHS_DISPLAYED - 1
if currentDate.month - (nbMonthsToRevwind%12) < 1 :
month = currentDate.month - (nbMonthsToRevwind%12)+12
if currentDate.month -nbMonthsToRevwind < 1:
year = currentDate.year - int( abs(math.floor( float( ( currentDate.month - nbMonthsToRevwind ) / 12 ) ) ) )
else :
month = currentDate.month - nbMonthsToRevwind
year = currentDate.year
start = "%s-%s-%s 00:00:00" %( year,month,"01" )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def getThreeClosestDatabasesBackups( infos ):
"""
@summary : Returns the three databases backups
that are the closest to the startTime
asked for the database recollection.
@param infos :
@return: the three databases backups
that are the closest to the startTime
asked for the database recollection.
"""
closestFiles = []
differenceFileTuples = []
files = os.listdir( StatsPaths.STATSDB + 'databasesTimeOfUpdatesBackups/' )
startTimeInEpochformat = StatsDateLib.getSecondsSinceEpoch( infos.databasesRecollectionStartTime )
for file in files:
#try:
fileDateInIsoFormat = "%s %s" %(str(file).split("_")[0], str(file).split("_")[1] )
tupleToAdd = ( abs( StatsDateLib.getSecondsSinceEpoch(fileDateInIsoFormat) - startTimeInEpochformat ), file )
differenceFileTuples.append( tupleToAdd )
for tuple in differenceFileTuples [:3] :
closestFiles.append( tuple[1] )
return closestFiles
def getStartAndEndTimeForPickleRecollection():
"""
@summary : Gets the start time and the endTime
of the pickle recollection from the
user's input.
@return : Returns the startTime and endTime.
"""
startTime = raw_input( "Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss) : ")
while not StatsDateLib.isValidIsoDate( startTime ):
print "Error. The entered date must be of the iso format."
startTime = raw_input( "Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss) : ")
endTime= raw_input( "Enter the endTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : ")
while( str(endTime).lower() != "now" and not StatsDateLib.isValidIsoDate( endTime ) and ( StatsDateLib.isValidIsoDate( endTime ) and endTime<= startTime ) ) :
if StatsDateLib.isValidIsoDate( endTime ) and endTime<= startTime :
print "Error. End time must be after startTime( %s ). "
elif StatsDateLib.isValidIsoDate( endTime ):
print "Error. The entered date must be of the iso format."
endTime= raw_input( "Enter the endTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : ")
if endTime == "now" :
endTime = StatsDateLib.getIsoFromEpoch( time.time() )
return startTime, endTime
def getTimeSeperatorsBasedOnAvailableMemory(startTime, endTime, clients,
fileType, machines):
"""
@summary: returns the time seperators to be used for the transfer
in a way that should prevent overloading memory.
@param startTime: start time of the transfer to be attempted.
@param endTime: end time of the transfer to be attempted.
@param clients: lists of clients/sources to be transferred.
@param fileType: tx or rx.
@param machines: machines on wich the clients/sources reside.
@return: the time seperators.
"""
width = 0 # Width in seconds of the transfer to be attempted
seperators = [
] # Time sperators representing every hour to be transferred.
allFiles = [] # List of all pickle files that will be involved
hourlyFiles = [] # List of all files to be handled for a certain hour.
hourlyFileSizes = [
] # Total file size of all the files to be handled at a certain hour.
totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory
currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine.
seperatorsBasedOnAvailableMemory = [
startTime, endTime
] # Suppose we have all the momory we need.
width = (StatsDateLib.getSecondsSinceEpoch(endTime) -
StatsDateLib.getSecondsSinceEpoch(startTime)) / StatsDateLib.HOUR
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=width * StatsDateLib.HOUR,
interval=StatsDateLib.HOUR)[:-1])
for seperator in seperators:
hourlyFiles = PickleMerging.createNonMergedPicklesList(
seperator, machines, fileType, clients)
allFiles.extend(hourlyFiles)
hourlyFileSizes.append(
MemoryManagement.getTotalSizeListOfFiles(hourlyFiles))
totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles(
allFiles)
currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory(
marginOfError=0.75
) #never expect more than 25% of the avaiable memory to be avaiable for pickle loading.
if totalSizeToloadInMemory >= currentlyAvailableMemory:
seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentlyAvailableMemory, hourlyFileSizes)
return seperatorsBasedOnAvailableMemory
def __init__( self, displayedLanguage = 'en', filesLanguage='en', days = None, \
weeks = None, months = None, years = None, \
pathsTowardsGraphics = None, pathsTowardsOutputFiles = None ):
"""
@summary : Constructor
@param displayedLanguage: Languages in which to display
the different captions found within
the generated web page.
@param fileLanguages: Language in which the files that
will be referenced within this page
have been generated.
@param days : List of days that the web page covers.
@note : Will set two global translators to be used throughout this module
_ which translates every caption that is to be printed.
_F which translates every filename that is to be linked.
"""
configParameters = StatsConfigParameters()
configParameters.getGeneralParametersFromStatsConfigurationFile()
global _
_ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, displayedLanguage )
if days == None:
self.setDays()
else:
self.days = days
if weeks == None:
self.setWeeks()
else:
self.weeks = weeks
if months == None:
self.setMonths()
else:
self.months = months
if years == None:
self.setYears()
else:
self.years = years
self.displayedLanguage = displayedLanguage
self.filesLanguage = filesLanguage
self.pathsTowardsGraphics = StatsPaths()
self.pathsTowardsGraphics.setPaths( filesLanguage )
self.pathsTowardsOutputFiles = StatsPaths()
self.pathsTowardsOutputFiles.setPaths( self.displayedLanguage )
StatsDateLib.setLanguage(filesLanguage)
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\
endTime = "2006-07-31 19:00:00", client = "satnet",\
machine = "pdsPM", fileType = "tx" ):
"""
@summary : This method merges entire hourly pickles files together.
@None : This does not support merging part of the data of pickles.
"""
if logger != None :
logger.debug( _("Call to mergeHourlyPickles received.") )
logging = True
else:
logging = False
pickles = []
entries = {}
width = StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime )
startTime = StatsDateLib.getIsoWithRoundedHours( startTime )
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=width, interval=60*StatsDateLib.MINUTE )[:-1])
for seperator in seperators :
pickles.append( StatsPickler.buildThisHoursFileName( client = client, offset = 0, currentTime = seperator, machine = machine, fileType = fileType ) )
startingNumberOfEntries = 0
#print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
for pickle in pickles :
if os.path.isfile( pickle ) :
tempCollection = CpickleWrapper.load( pickle )
if tempCollection != None :
for i in xrange( len( tempCollection.fileEntries ) ):
entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i]
startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries )
else:
sys.exit()
else:
emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60, entries = {} )
for i in xrange( 60 ):
entries[i + startingNumberOfEntries ] = emptyEntries [i]
startingNumberOfEntries = startingNumberOfEntries + 60
#print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) )
statsCollection = FileStatsCollector( startTime = startTime , endTime = endTime, interval = StatsDateLib.MINUTE, totalWidth = width, fileEntries = entries,fileType= fileType, logger = logger, logging = logging )
return statsCollection
def askUserAboutUpdatingLogs( infos ):
"""
@Summary : Asks user about whether or not
he wants to update the log files
on his machine.
@returns True or False
"""
updateLofFiles = False
os.system( "clear" )
showPresentation()
print ""
print ""
print "***************** Important note *****************"
print "Collection or recollection of pickle files "
print "is closely linked to the log files found on this machine."
if StatsDateLib.getIsoWithRoundedHours( infos.picklesRecollectionStartTime ) != StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( time.time() )) :
print "Data recollection is set to take place up to the current hour."
print "For the end of the recollection it is recommended that log file be updated."
print "However, if the recollection spans over a long while and that the log file currently "
print "on this machine are 'old' and cover the start of the recollection,"
print "updating log files might cause you to loose some or all of those old files."
else :
print "Data recollection is set to end PRIOR to the current hour."
print "In this case, log file updates are usually useless."
print "In the case where the span between the start of the recollection "
print "is longer than the span covered by the currently accessible log files, "
print "usefull log files will be lsot by updating them."
print "However the opposite could also be true. If problems occured and "
print "databases are seriously outdated, updating them will be the only solution "
print "capable of making some or all the needed log file data accessible for pickling."
print ""
print "***Please review log files prior to specifying whether or not you want to update them or not.***"
print ""
input = raw_input( "Do you want to update log files ? ( y or n ) : " )
while ( str( input ).lower() != 'n' and str( input ).lower() != 'y' ):
print "Please enter one of the following choices : y/Y or n/N."
input = raw_input( "Do you want to update log files ? ( y or n ) : " )
if str( input ).lower() == 'y' :
print "Log files will be updated."
updateLofFiles = True
else:
print "Log files will not be updated."
return updateLofFiles
def askUserAboutUpdatingLogs(infos):
"""
@Summary : Asks user about whether or not
he wants to update the log files
on his machine.
@returns True or False
"""
updateLofFiles = False
os.system("clear")
showPresentation()
print ""
print ""
print "***************** Important note *****************"
print "Collection or recollection of pickle files "
print "is closely linked to the log files found on this machine."
if StatsDateLib.getIsoWithRoundedHours(
infos.picklesRecollectionStartTime
) != StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(time.time())):
print "Data recollection is set to take place up to the current hour."
print "For the end of the recollection it is recommended that log file be updated."
print "However, if the recollection spans over a long while and that the log file currently "
print "on this machine are 'old' and cover the start of the recollection,"
print "updating log files might cause you to loose some or all of those old files."
else:
print "Data recollection is set to end PRIOR to the current hour."
print "In this case, log file updates are usually useless."
print "In the case where the span between the start of the recollection "
print "is longer than the span covered by the currently accessible log files, "
print "usefull log files will be lsot by updating them."
print "However the opposite could also be true. If problems occured and "
print "databases are seriously outdated, updating them will be the only solution "
print "capable of making some or all the needed log file data accessible for pickling."
print ""
print "***Please review log files prior to specifying whether or not you want to update them or not.***"
print ""
input = raw_input("Do you want to update log files ? ( y or n ) : ")
while (str(input).lower() != 'n' and str(input).lower() != 'y'):
print "Please enter one of the following choices : y/Y or n/N."
input = raw_input("Do you want to update log files ? ( y or n ) : ")
if str(input).lower() == 'y':
print "Log files will be updated."
updateLofFiles = True
else:
print "Log files will not be updated."
return updateLofFiles
def getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, fileSizesPerHour, usage= "rrd" ):
"""
@summary : returns a list of time seperators based on a list of file and
the current amount of free memory. Each seperator represents the time
associated with a certain hourly file. Each seperator will represent
the maximum amount of files that can be treated at the same time
without busting the current memory.
@attention: List fo files MUST refer to hourly files.
@param startTime: Startime in iso format of the interval to work with.
@param endTime: End time in iso format of the interval to work with.
@param currentFreeMemory: Maximum amout of memory to use per seperation.
@param fileSizesPerHour: size of the file(s) to be treated at every hour.
@return: Returns the time seperators.
"""
currentTotalFileSizes = 0
currentTime = StatsDateLib.getSecondsSinceEpoch(startTime)
seperators = [startTime]
if fileSizesPerHour[0] < currentFreeMemory:
for fileSizePerHour in fileSizesPerHour :
currentTotalFileSizes = currentTotalFileSizes + fileSizePerHour
if currentFreeMemory < currentTotalFileSizes:
seperators.append( StatsDateLib.getIsoFromEpoch(currentTime))
currentTotalFileSizes = 0
currentTime = currentTime + StatsDateLib.HOUR
else:
raise Exception( "Cannot build seperators. First file will not even fit within current available memory." )
if seperators[len(seperators) -1 ] != endTime :
seperators.append( endTime )
if len(seperators) > 2 : #If any "in between seperators were added"
i = 1
currentLength = len(seperators) -1
while i < currentLength: #add 1 minute
if usage == "rrd":
seperators.insert(i+1, StatsDateLib.getIsoFromEpoch( (StatsDateLib.getSecondsSinceEpoch(seperators[i]) + StatsDateLib.MINUTE)))
else:
seperators.insert( i+1, StatsDateLib.getSecondsSinceEpoch(seperators[i]) )
currentLength = currentLength + 1
i = i + 2
return seperators
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() - (120 * 60)
endTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat))
startTime = StatsDateLib.getIsoWithRoundedHours(
StatsDateLib.getIsoFromEpoch(currentTimeEpochFormat -
(StatsDateLib.DAY * 7)))
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[groupName]
machines = statsConfig.groupParameters.groupsMachines[groupName]
fileType = statsConfig.groupParameters.groupFileTypes[groupName]
seperators = [startTime]
seperators.extend(
StatsDateLib.getSeparatorsWithStartTime(
startTime=startTime,
width=StatsDateLib.DAY * 7,
interval=StatsDateLib.HOUR)[:-1])
listOfFiles = PickleMerging.createMergedPicklesList(
startTime, endTime, clients, groupName, fileType, machines, seperators)
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles(
listOfFiles) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments(
startTime, endTime, currentFreeMemory, listOfFileSizes)
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" % (
currentFreeMemory, getTotalSizeListOfFiles(listOfFiles))
print "we have enough memory to merge all these pickles."
def buildThisHoursFileName(client="someclient",
offset=0,
currentTime="",
fileType="tx",
machine="someMachineName"):
"""
@summary : Builds a filename using current currentTime.
@Note : The format will be something like this :
StatsPaths.STATSPICKLES/clientName/date/TXorRX//machine_hour
Ex : StatsPaths.STATSPICKLES/clientName/20060707/tx/machinex_12:00:00
offset can be used to find a file from an hour close to the current one
tempcurrentTime can also be used to build a filename from another hour.
@warning :To be used only with pickles created hourly.
"""
timeFolder = ""
if currentTime == "":
currentTime = time.time()
else:
currentTime = StatsDateLib.getSecondsSinceEpoch(currentTime)
currentTime = currentTime + (offset * StatsDateLib.HOUR)
splitTime = time.gmtime(currentTime)
for i in range(3):
if int(splitTime[i]) < 10:
timeFolder = timeFolder + "0" + str(splitTime[i])
else:
timeFolder = timeFolder + str(splitTime[i])
hour = StatsDateLib.getHoursFromIso(
StatsDateLib.getIsoFromEpoch(currentTime))
maxLt = (os.statvfs(STATSPATHS.STATSPICKLES)[statvfs.F_NAMEMAX])
fileName = ("%s" + "%." + str(maxLt) + "s/%s/%s/%." + str(maxLt) +
"s_%s") % (STATSPATHS.STATSPICKLES, client, timeFolder,
fileType, str(machine), str(hour))
return fileName
def printPickledTimes( pickledTimes ):
"""
@summary: Prints out all the pickled times found.
@param pickledTimes: Dictionary containing containing the
name -> timeOfUpdate relationships.
"""
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch(currentTime)
keys = pickledTimes.keys()
keys.sort()
os.system( 'clear' )
print "######################################################################"
print "# List of current times of updates. #"
print "# Times were found at : %-43s #" %currentTime
print "# On the machine named : %-43s #"%LOCAL_MACHINE
for key in keys:
print("#%32s : %33s#") %( key, pickledTimes[key] )
print "# #"
print "######################################################################"
def isFirstUpdateOfTheYear( self, timeOfUpdateInIsoFormat = "" ):
"""
@summary : Returns whether or not an update executed at
timeOfUpdateInIsoFormat would be the first update
of the year.
@timeOfUpdateInIsoFormat : Time at which the update would be executed.
@return : True or False.
"""
isFirstUpdateOfTheYear = False
lastUpdateISO = self.getTimeOfLastUpdateInLogs()
if timeOfUpdateInIsoFormat == "" :
timeOfUpdateInIsoFormat = StatsDateLib.getCurrentTimeInIsoformat()
if timeOfUpdateInIsoFormat > lastUpdateISO :
yearNumberOfLastUpdate = lastUpdateISO.split("-")[0]
yearNumberOfCurrentUpdate = timeOfUpdateInIsoFormat.split("-")[0]
if yearNumberOfLastUpdate != yearNumberOfCurrentUpdate:
isFirstUpdateOfTheYear = True
return isFirstUpdateOfTheYear
def __generateAllMissingDailyGraphicsSinceLasteUpdate(
self, generateTotalsGraphics):
"""
@summary : generates the daily graphics that were not generated between
last update and timeOfRequest.
@param generateTotalsGraphics: Whether or not to generate the totals graphics.
"""
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
updateManager = AutomaticUpdatesManager(
configParameters.nbAutoUpdatesLogsToKeep, "pxStatsStartup")
missingDays = updateManager.getMissingDaysBetweenUpdates(
updateManager.getTimeOfLastUpdateInLogs(), self.timeOfRequest)
missingDays.append(self.timeOfRequest)
oldTimeOfRequest = self.timeOfRequest
for missingDay in missingDays[1:]:
self.timeOfRequest = StatsDateLib.getIsoTodaysMidnight(missingDay)
self.__generateAllForDailyWebPage(False, generateTotalsGraphics)
self.__generateAllGraphicsForGroups("daily")
self.timeOfRequest = oldTimeOfRequest
def addOptions( parser ):
"""
@summary: This method is used to add all available options to the option parser.
"""
parser.add_option("-c", "--clients", action="store", type="string", dest="clients", default="ALL",
help=_("Clients' names"))
parser.add_option("-d", "--daily", action="store_true", dest = "daily", default=False, help=_("Create csv file containing daily data.") )
parser.add_option( "--date", action="store", type="string", dest="date", default=StatsDateLib.getIsoFromEpoch( time.time() ), help=_("Decide end time of graphics. Usefull for testing.") )
parser.add_option("-f", "--fileType", action="store", type="string", dest="fileType", default='tx', help=_("Type of log files wanted.") )
parser.add_option( "--fixedPrevious", action="store_true", dest="fixedPrevious", default=False, help=_("Do not use floating weeks|days|months|years. Use previous fixed interval found.") )
parser.add_option( "--fixedCurrent", action="store_true", dest="fixedCurrent", default=False, help=_("Do not use floating weeks|days|months|years. Use current fixed interval found.") )
parser.add_option( "--includeGroups", action="store_true", dest="includeGroups", default=False, help=_("Include groups of all the specified machines or clusters." ) )
parser.add_option( "-l", "--language", action="store", type="string", dest="outputLanguage", default="", help = _("Language in which you want the casv file to be created in." ) )
parser.add_option( "--machines", action="store", type="string", dest="machines", default=LOCAL_MACHINE, help =_("Machines for wich you want to collect data." ) )
parser.add_option("--machinesAreClusters", action="store_true", dest = "machinesAreClusters", default=False, help=_("Specified machines are clusters.") )
parser.add_option("-m", "--monthly", action="store_true", dest = "monthly", default=False, help=_("Create csv file containing monthly data." ) )
parser.add_option("--turnOffLogging", action="store_true", dest = "turnOffLogging", default=False, help=_("Turn off the logger") )
parser.add_option("-w", "--weekly", action="store_true", dest = "weekly", default=False, help=_("Create csv file containing weekly data." ) )
parser.add_option("-y", "--yearly", action="store_true", dest = "yearly", default=False, help=_("Create csv file containing yearly data." ) )
def getPreviousMonitoringJob( self, currentTime ):
"""
@summary : Gets the previous crontab from the pickle file.
@return : Time of the previous monitoring job.
@warning : Returns "" if file does not exist.
"""
statsPaths = StatsPaths()
statsPaths.setPaths()
file = "%spreviousMonitoringJob" %statsPaths.STATSMONITORING
previousMonitoringJob = ""
if os.path.isfile( file ):
fileHandle = open( file, "r" )
previousMonitoringJob = pickle.load( fileHandle )
fileHandle.close()
#print previousMonitoringJob
else:
previousMonitoringJob = StatsDateLib.getIsoTodaysMidnight( currentTime )
#print previousMonitoringJob
return previousMonitoringJob
def printPickledTimes(pickledTimes):
"""
@summary: Prints out all the pickled times found.
@param pickledTimes: Dictionary containing containing the
name -> timeOfUpdate relationships.
"""
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch(currentTime)
keys = pickledTimes.keys()
keys.sort()
os.system("clear")
print "######################################################################"
print "# List of current times of updates. #"
print "# Times were found at : %-43s #" % currentTime
print "# On the machine named : %-43s #" % LOCAL_MACHINE
for key in keys:
print ("#%32s : %33s#") % (key, pickledTimes[key])
print "# #"
print "######################################################################"
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : Start,end tuple both in ISO format.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
start = StatsDateLib.rewindXDays( currentTime, NB_DAYS_DISPLAYED - 1 )
start = StatsDateLib.getIsoTodaysMidnight( start )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def getStartEndOfWebPage():
"""
@summary : Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
@return : Start,end tuple both in ISO format.
"""
currentTime = StatsDateLib.getIsoFromEpoch(time.time())
start = StatsDateLib.rewindXDays(currentTime, NB_DAYS_DISPLAYED - 1)
start = StatsDateLib.getIsoTodaysMidnight(start)
end = StatsDateLib.getIsoTodaysMidnight(currentTime)
return start, end
def getStartEndOfWebPage():
"""
Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
"""
currentTime = StatsDateLib.getIsoFromEpoch(time.time())
start = StatsDateLib.rewindXDays(currentTime, (NB_YEARS_DISPLAYED - 1) * 365)
start = StatsDateLib.getIsoTodaysMidnight(start)
end = StatsDateLib.getIsoTodaysMidnight(currentTime)
return start, end
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch(currentTime)
currentTime = StatsDateLib.getIsoWithRoundedSeconds(currentTime)
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len(sys.argv) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime(
timeToRestore,
'%Y-%m-%d %H:%M:%S') #will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" % (split[0], split[1])
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
restoreDatabases(timeToRestore, currentTime,
generalParameters.nbDbBackupsToKeep)
else:
print _("You must specify a date.")
print _("Date must be of the folowing format YYYY-MM-DD HH:MM:SS")
print _("Program terminated.")
def getStartEndOfWebPage():
"""
Returns the time of the first
graphics to be shown on the web
page and the time of the last
graphic to be displayed.
"""
currentTime = StatsDateLib.getIsoFromEpoch( time.time() )
start = StatsDateLib.rewindXDays( currentTime, ( NB_YEARS_DISPLAYED - 1 ) * 365 )
start = StatsDateLib.getIsoTodaysMidnight( start )
end = StatsDateLib.getIsoTodaysMidnight( currentTime )
return start, end
def addFilecountLabelsToGraph( self, clientIndex, statsTypeIndex, nbGraphs, maxPairValue ):
"""
@summary: Used to set proper labels for a graph relating to bytes.
"""
_ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.outputLanguage )
if self.maximums[clientIndex][statsTypeIndex] !=None and self.maximums[clientIndex][statsTypeIndex] != 0 :
timeOfMax = self.timeOfMax[clientIndex][statsTypeIndex]
timeOfMax = StatsDateLib.getIsoWithRoundedSeconds( timeOfMax )
if maxPairValue < 5 :
self.graph( 'set format y "%7.2f"' )
else:
self.graph( 'set format y "%7.0f"' )
maximum = self.maximums[clientIndex][statsTypeIndex]
else:
timeOfMax = ""
maximum = ""
if self.groupName == "" :
entityType = self.sourlient
if self.groupName == "" :
entityType = self.sourlient
if len( self.stats ) ==1:
entityName = str(self.clientNames).replace("[","").replace("]","").replace("'","")
else:
entityName = self.clientNames[clientIndex]
else:
entityType = "Group"
entityName = self.groupName
formatedProductTypes = self.getFormatedProductTypesForLabel()
self.graph( 'set size .545, .37' )
self.graph( 'set origin 0, %3.2f' %( ((nbGraphs)*.40) ))
self.graph.ylabel( _('Files/Minute') )
self.graph( 'set label "%s : %s" at screen .545, screen %3.2f' % ( entityType, entityName,(.26+(nbGraphs) *.40) ))
self.graph( 'set label '+_('"Machines : %s"')%self.machines + ' at screen .545, screen %3.2f' % ( (.24+(nbGraphs) *.40) ) )
self.graph( 'set label '+_('"Product type(s) : %s"')%formatedProductTypes +' at screen .545, screen %3.2f' % ( (.22+(nbGraphs) *.40) ) )
self.graph( 'set label '+_('"Max files/%s : %s"')%(self.entryType, maximum) +' at screen .545, screen %3.2f' % ( (.20+(nbGraphs) *.40) ))
self.graph( 'set label '+_('"Time of max. : %s"')%timeOfMax +' at screen .545, screen %3.2f' % ( ( (.18+(nbGraphs) *.40) )))
self.graph( 'set label '+_('"# of files : %s"')%self.nbFiles[clientIndex] + ' at screen .545, screen %3.2f' % ( (.16+(nbGraphs) *.40) ) )
def filterClientsNamesUsingWilcardFilters(currentTime, timespan,
clientNames, machines,
fileTypes):
"""
@param currentTime: currentTime specified in the parameters.
@param timespan: Time span specified within the parameters.
@param clientNames:List of client names found in the parameters.
"""
newClientNames = []
end = currentTime
start = StatsDateLib.getIsoFromEpoch(
StatsDateLib.getSecondsSinceEpoch(currentTime) -
60 * 60 * timespan)
if len(clientNames) >= len(fileTypes) or len(fileTypes) == 1:
if len(fileTypes) == 1:
for i in range(1, len(clientNames)):
fileTypes.append(fileTypes[0])
for clientName, fileType in map(None, clientNames, fileTypes):
if '?' in clientName or '*' in clientName:
pattern = clientName
rxHavingRun, txHavingRun = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(
start, end, machines, pattern)
if fileType == "rx":
namesHavingrun = rxHavingRun
else:
namesHavingrun = txHavingRun
newClientNames.extend(namesHavingrun)
else:
newClientNames.append(clientName)
return newClientNames
def getTimeSeperatorsBasedOnAvailableMemory( startTime, endTime, clients, fileType, machines ):
"""
@summary: returns the time seperators to be used for the transfer
in a way that should prevent overloading memory.
@param startTime: start time of the transfer to be attempted.
@param endTime: end time of the transfer to be attempted.
@param clients: lists of clients/sources to be transferred.
@param fileType: tx or rx.
@param machines: machines on wich the clients/sources reside.
@return: the time seperators.
"""
width = 0 # Width in seconds of the transfer to be attempted
seperators = [] # Time sperators representing every hour to be transferred.
allFiles =[] # List of all pickle files that will be involved
hourlyFiles = [] # List of all files to be handled for a certain hour.
hourlyFileSizes = [] # Total file size of all the files to be handled at a certain hour.
totalSizeToloadInMemory = 0.0 # Total size of all the pickle files to load in memory
currentlyAvailableMemory = 0.0 # Total currently available memory on the present machine.
seperatorsBasedOnAvailableMemory = [startTime, endTime] # Suppose we have all the momory we need.
width = ( StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) ) / StatsDateLib.HOUR
seperators = [ startTime ]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width= width*StatsDateLib.HOUR, interval=StatsDateLib.HOUR )[:-1])
for seperator in seperators:
hourlyFiles = PickleMerging.createNonMergedPicklesList( seperator, machines, fileType, clients )
allFiles.extend( hourlyFiles )
hourlyFileSizes.append( MemoryManagement.getTotalSizeListOfFiles( hourlyFiles ) )
totalSizeToloadInMemory = MemoryManagement.getTotalSizeListOfFiles( allFiles )
currentlyAvailableMemory = MemoryManagement.getCurrentFreeMemory( marginOfError = 0.75 )#never expect more than 25% of the avaiable memory to be avaiable for pickle loading.
if totalSizeToloadInMemory >= currentlyAvailableMemory:
seperatorsBasedOnAvailableMemory = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentlyAvailableMemory, hourlyFileSizes )
return seperatorsBasedOnAvailableMemory
def prepareQuery(self):
"""
@summary : Buildup the query to be executed.
@SIDE_EFFECT : modifies self.query value.
"""
global _
if self.queryParameters.combine == 'true':
totals = True
mergerType = "regular"
else:
totals = False
mergerType = ""
fixedCurrent = False
fixedPrevious = False
if _("current") in str(self.queryParameters.fixedSpan).lower() :
fixedCurrent = True
elif _("previous") in str(self.queryParameters.fixedSpan).lower():
fixedPrevious = True
else:
fixedCurrent = False
fixedPrevious = False
hour = self.queryParameters.endTime.split(" ")[1]
splitDate = self.queryParameters.endTime.split(" ")[0].split( '-' )
date = splitDate[2] + '-' + splitDate[1] + '-' + splitDate[0] + " " + hour
if self.queryParameters.span == "":
timespan = 0
else:
timespan = int(self.queryParameters.span )
StatsDateLib.setLanguage( self.querierLanguage )
startTime, endTime = StatsDateLib.getStartEndInIsoFormat(date, timespan, self.queryParameters.specificSpan, fixedCurrent, fixedPrevious )
timespan = int( StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) ) / 3600
combinedMachineName = ""
for machine in self.queryParameters.machines:
combinedMachineName = combinedMachineName + machine
machines = [ combinedMachineName ]
self.graphicProducer = RRDGraphicProducer( self.queryParameters.fileTypes[0], self.queryParameters.statsTypes ,\
totals, self.queryParameters.specificSpan,\
self.queryParameters.sourLients, timespan,\
startTime, endTime, machines, False,
mergerType, True, self.querierLanguage, self.querierLanguage )
StatsDateLib.setLanguage( LanguageTools.getMainApplicationLanguage() )
def main():
"""
@summary : This program is to be used to backup rrd databases and their corresponding
time of update files. Backing up rrd databases at various point in time is a
recommended paractice in case newly entered data is not valid.
"""
setGlobalLanguageParameters()
timeToRestore = "2006-10-23 09:00:00"
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentTime = StatsDateLib.getIsoWithRoundedSeconds( currentTime )
currentTime = currentTime.replace(" ", "_")
generalParameters = StatsConfigParameters()
generalParameters.getAllParameters()
if len( sys.argv ) == 2:
print sys.argv
#try:
timeToRestore = sys.argv[1]
t = time.strptime( timeToRestore, '%Y-%m-%d %H:%M:%S' )#will raise exception if format is wrong.
split = timeToRestore.split()
timeToRestore = "%s_%s" %( split[0], split[1] )
# except:
# print 'Date must be of the following format "YYYY-MM-DD HH:MM:SS"'
# print "Program terminated."
# sys.exit()
restoreDatabaseUpdateTimes( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
restoreDatabases( timeToRestore, currentTime, generalParameters.nbDbBackupsToKeep )
else:
print _( "You must specify a date." )
print _( "Date must be of the folowing format YYYY-MM-DD HH:MM:SS" )
print _( "Program terminated." )
def addOptions(parser):
"""
This method is used to add all available options to the option parser.
"""
parser.add_option("-c",
"--clients",
action="store",
type="string",
dest="clients",
default=_("ALL"),
help=_("Clients for wich we need to tranfer the data."))
parser.add_option("-e",
"--end",
action="store",
type="string",
dest="end",
default=StatsDateLib.getIsoFromEpoch(time.time()),
help=_("Decide ending time of the update."))
parser.add_option("-f",
"--fileTypes",
action="store",
type="string",
dest="fileTypes",
default="",
help=_("Specify the data type for each of the clients."))
parser.add_option(
"-g",
"--group",
action="store",
type="string",
dest="group",
default="",
help=
_("Transfer the combined data of all the specified clients/sources into a grouped database."
))
parser.add_option("-m",
"--machines",
action="store",
type="string",
dest="machines",
default=LOCAL_MACHINE,
help=_("Specify on wich machine the clients reside."))
parser.add_option("-p",
"--products",
action="store",
type="string",
dest="products",
default=_("ALL"),
help=_("Specify wich product you are interested in."))
def isFirstUpdateOfTheWeek( self, timeOfUpdateInIsoFormat = "" ):
"""
@summary : Returns whether or not an update executed at
timeOfUpdateInIsoFormat would be the first update
of the week.
@timeOfUpdateInIsoFormat : Time at which the update would be executed.
@return : True or False.
"""
isFirstUpdateOfTheWeek = False
lastUpdateISO = self.getTimeOfLastUpdateInLogs()
if timeOfUpdateInIsoFormat == "" :
timeOfUpdateInIsoFormat = StatsDateLib.getCurrentTimeInIsoformat()
if timeOfUpdateInIsoFormat > lastUpdateISO :
lastUpdateDT = datetime( int( lastUpdateISO.split("-")[0]),\
int( lastUpdateISO.split("-")[1]),\
int( lastUpdateISO.split("-")[1].split(" ")[0] )\
)
currentUpdateDT = datetime( int( timeOfUpdateInIsoFormat.split("-")[0]),\
int( timeOfUpdateInIsoFormat.split("-")[1]),\
int( timeOfUpdateInIsoFormat.split("-")[1].split(" ")[0] )\
)
weekNumberOfLastUpdate = time.strftime( '%W', time.gmtime( StatsDateLib.getSecondsSinceEpoch( lastUpdateISO ) ) )
weekNumberOfCurrentUpdate = time.strftime( '%W', time.gmtime( StatsDateLib.getSecondsSinceEpoch( timeOfUpdateInIsoFormat ) ) )
timeBetweenBothDates = currentUpdateDT - lastUpdateDT
daysBetween = timeBetweenBothDates.days
if daysBetween < 7 and ( weekNumberOfLastUpdate == weekNumberOfCurrentUpdate ): #<7 days prevents same week but from different years.
isFirstUpdateOfTheWeek = False
else:
isFirstUpdateOfTheWeek = True
return isFirstUpdateOfTheWeek
def getParametersFromMonitoringConfigurationFile( self ):
"""
@summary : Gather all the parameters from the StatsPaths.STATSETC/config file.
@return : All collected values in this order emails, machines,
files, folders, maxUsages, errorsLogFile, maxSettingsFile.
"""
statsPaths = StatsPaths()
statsPaths.setPaths()
CONFIG = statsPaths.STATSETC +"monitoringConf"
config = ConfigParser()
if os.path.isfile( CONFIG ):
file = open( CONFIG )
config.readfp( file )
self.emails = config.get( 'statsMonitoring', 'emails' ).split( ";" )
self.sender = config.get( 'statsMonitoring', 'sender' )
self.smtpServer = config.get( 'statsMonitoring', 'smtpServer' )
self.machines = config.get( 'statsMonitoring', 'machines' ).split( ";" )
self.files = config.get( 'statsMonitoring', 'files' ).split( ";" )
self.folders = config.get( 'statsMonitoring', 'folders' ).split( ";" )
self.maxUsages = config.get( 'statsMonitoring', 'maxUsages' ).split( ";" )
self.errorsLogFile = config.get( 'statsMonitoring', 'errorsLogFile' )
self.maxSettingsFile=config.get( 'statsMonitoring', 'maxSettingsFile' )
self.endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( time.time() ) )
self.startTime = self.getPreviousMonitoringJob(self.endTime)
self.maximumGaps = self.getMaximumGaps( )
self.updateMachineNamesBasedOnExistingMachineTags()
try:
file.close()
except:
pass
else:
#print "%s configuration file not present. Please restore file prior to running" %CONFIG
raise Exception( "%s configuration file not present. Please restore file prior to running" %CONFIG )
def buildThisHoursFileName( client = "someclient", offset = 0, currentTime = "", fileType = "tx", machine = "someMachineName" ):
"""
@summary : Builds a filename using current currentTime.
@Note : The format will be something like this :
StatsPaths.STATSPICKLES/clientName/date/TXorRX//machine_hour
Ex : StatsPaths.STATSPICKLES/clientName/20060707/tx/machinex_12:00:00
offset can be used to find a file from an hour close to the current one
tempcurrentTime can also be used to build a filename from another hour.
@warning :To be used only with pickles created hourly.
"""
timeFolder = ""
if currentTime == "":
currentTime = time.time()
else:
currentTime = StatsDateLib.getSecondsSinceEpoch( currentTime )
currentTime = currentTime + ( offset * StatsDateLib.HOUR )
splitTime = time.gmtime( currentTime )
for i in range( 3 ):
if int( splitTime[i] ) < 10 :
timeFolder = timeFolder + "0" + str( splitTime[i] )
else:
timeFolder = timeFolder + str( splitTime[i] )
hour = StatsDateLib.getHoursFromIso( StatsDateLib.getIsoFromEpoch( currentTime ) )
maxLt = ( os.statvfs( STATSPATHS.STATSPICKLES )[statvfs.F_NAMEMAX])
fileName = ( "%s" + "%." + str( maxLt ) + "s/%s/%s/%." + str( maxLt ) + "s_%s" ) %( STATSPATHS.STATSPICKLES, client, timeFolder, fileType, str(machine), str(hour) )
return fileName
def setYears( self ):
"""
Returns the last 3 year numbers including the current year.
"""
currentTime = time.time()
currentTime = StatsDateLib.getIsoFromEpoch( currentTime )
currentDate = datetime.date( int(currentTime[0:4]), int(currentTime[5:7]), 1 )
years = []
for i in range(0,3):
year = currentDate.year - i
newDate = StatsDateLib.getSecondsSinceEpoch( "%s-%s-%s 00:00:00" %(year, currentDate.month, currentDate.day) )
years.append( newDate )
years.reverse()
self.years = years
def filterClientsNamesUsingWilcardFilters( currentTime, timespan, clientNames, machines, fileTypes ):
"""
@param currentTime: currentTime specified in the parameters.
@param timespan: Time span specified within the parameters.
@param clientNames:List of client names found in the parameters.
"""
newClientNames = []
end = currentTime
start = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentTime)- 60*60*timespan )
if len(clientNames) >= len( fileTypes ) or len( fileTypes ) ==1:
if len( fileTypes ) == 1 :
for i in range(1, len( clientNames ) ):
fileTypes.append( fileTypes[0])
for clientName,fileType in map( None, clientNames, fileTypes ):
if '?' in clientName or '*' in clientName :
pattern =clientName
rxHavingRun,txHavingRun = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod(start, end, machines, pattern)
if fileType == "rx":
namesHavingrun = rxHavingRun
else:
namesHavingrun = txHavingRun
newClientNames.extend( namesHavingrun )
else:
newClientNames.append( clientName )
return newClientNames
def getMissingWeeksBetweenUpdates(self, update1InIsoFormat, update2InIsoFormat ):
"""
@summary : Returns the list of days between update date 1 and update date 2.
@Note : If update1InIsoFormat = 2008-02-28 15:00:00 and
update2InIsoFormat = 2008-02-28 15:00:00
this method would return [ 2008-02-28 15:00:00 ]
@return : Returns the list of days between update date 1 and update date 2.
"""
missingWeeks = []
if update2InIsoFormat > update1InIsoFormat:
weekInIsoFormat = update1InIsoFormat
while weekInIsoFormat <= update2InIsoFormat :
missingWeeks.append( weekInIsoFormat )
weekInIsoFormat = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( weekInIsoFormat ) + ( StatsDateLib.DAY*7 ) )
return missingWeeks[:-1]
def getStartTimeAndEndTime( self, collectUptoNow = False ):
"""
@summary : Returns the startTime and endTime of the graphics.
@warning : collectUptoNow not yet supported in program !
@return : the startTime and endTime of the graphics.
"""
#Now not yet implemented.
if collectUptoNow == True :
endTime = self.currentTime
else :
endTime = StatsDateLib.getIsoWithRoundedHours( self.currentTime )
startTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( endTime ) - (self.timespan * StatsDateLib.HOUR) )
return startTime, endTime
def getLastUpdate( machine, client, fileType, currentDate, paths, collectUpToNow = False ):
"""
@summary : Reads and returns the client's or source's last update.
@return : The client's or sources last update.
"""
times = {}
lastUpdate = {}
fileName = "%s%s_%s_%s" %( paths.STATSPICKLESTIMEOFUPDATES, fileType, client, machine )
if os.path.isfile( fileName ):
try :
fileHandle = open( fileName, "r" )
lastUpdate = pickle.load( fileHandle )
fileHandle.close()
except:
print _("problematic file in loading : %s") %fileName
lastUpdate = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentDate ) - StatsDateLib.HOUR) )
pass
fileHandle.close()
else:#create a new pickle file.Set start of the pickle as last update.
if not os.path.isdir( os.path.dirname( fileName ) ) :
os.makedirs( os.path.dirname( fileName ) )
fileHandle = open( fileName, "w" )
lastUpdate = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentDate ) - StatsDateLib.HOUR) )
pickle.dump( lastUpdate, fileHandle )
fileHandle.close()
return lastUpdate
def getStartTimeAndEndTime(self, collectUptoNow=False):
"""
@summary : Returns the startTime and endTime of the graphics.
@warning : collectUptoNow not yet supported in program !
@return : the startTime and endTime of the graphics.
"""
#Now not yet implemented.
if collectUptoNow == True:
endTime = self.currentTime
else:
endTime = StatsDateLib.getIsoWithRoundedHours(self.currentTime)
startTime = StatsDateLib.getIsoFromEpoch(
StatsDateLib.getSecondsSinceEpoch(endTime) -
(self.timespan * StatsDateLib.HOUR))
return startTime, endTime
def getXTics( self ):
"""
@summary : This method builds all the xtics used to seperate data on the x axis.
Xtics values will are used in the plot method so they will be drawn on
the graphic.
@note: All xtics will be devided hourly. This means a new xtic everytime
another hour has passed since the starting point.
"""
_ = self.getTranslatorForModule( CURRENT_MODULE_ABS_PATH, self.workingLanguage )
#print "get x tics"
if self.logger != None :
self.logger.debug( _("Call to getXtics received") )
nbBuckets = ( len( self.stats[0].statsCollection.timeSeperators ) )
xtics = ''
startTime = StatsDateLib.getSecondsSinceEpoch( self.stats[0].statsCollection.timeSeperators[0] )
if nbBuckets != 0 :
for i in range(0, nbBuckets ):
if ( ( StatsDateLib.getSecondsSinceEpoch(self.stats[0].statsCollection.timeSeperators[i]) - ( startTime ) ) %(60*60) == 0.0 ):
hour = StatsDateLib.getHoursFromIso( self.stats[0].statsCollection.timeSeperators[i] )
xtics += '"%s" %i, '%( hour , StatsDateLib.getSecondsSinceEpoch(self.stats[0].statsCollection.timeSeperators[i] ) )
#print nbBuckets
#print "len xtics %s" %len(xtics)
return xtics[:-2]
def main():
"""
@summary: Small test case to see if everything works fine
"""
statsConfig = StatsConfigParameters()
statsConfig.getAllParameters()
machineconfig = MachineConfigParameters()
machineconfig.getParametersFromMachineConfigurationFile()
currentTimeEpochFormat = time.time() -(120*60)
endTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat ) )
startTime = StatsDateLib.getIsoWithRoundedHours( StatsDateLib.getIsoFromEpoch( currentTimeEpochFormat -( StatsDateLib.DAY*7 ) ) )
print startTime, endTime
groupName = statsConfig.groupParameters.groups[0]
clients = statsConfig.groupParameters.groupsMembers[ groupName ]
machines = statsConfig.groupParameters.groupsMachines[ groupName ]
fileType = statsConfig.groupParameters.groupFileTypes[ groupName ]
seperators = [startTime]
seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=StatsDateLib.DAY*7, interval=StatsDateLib.HOUR )[:-1])
listOfFiles = PickleMerging.createMergedPicklesList( startTime, endTime, clients, groupName, fileType, machines, seperators )
listOfFileSizes = MemoryManagement.getListOfFileSizes(listOfFiles)
currentFreeMemory = MemoryManagement.getCurrentFreeMemory(0.55555)
if MemoryManagement.getTotalSizeListOfFiles( listOfFiles ) > currentFreeMemory:
seperators = MemoryManagement.getSeperatorsForHourlyTreatments( startTime, endTime, currentFreeMemory, listOfFileSizes )
print seperators
else:
print "We have %s bytes free and the pickles require %s bytes" %( currentFreeMemory, getTotalSizeListOfFiles( listOfFiles ) )
print "we have enough memory to merge all these pickles."
def buildCsvFileName(infos):
"""
@summary: Builds and returns the file name to use for the csv file.
@param infos: _CvsInfos instance containing the required
information to build up the file name.
@return: Return the built up file name.
"""
global _
StatsDateLib.setLanguage(infos.outputLanguage)
paths = StatsPaths()
paths.setPaths(infos.outputLanguage)
machinesStr = str(infos.machinesForLabels).replace('[', '').replace(
']', '').replace(',', '').replace("'",
"").replace('"',
'').replace(' ', '')
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(
StatsDateLib.getSecondsSinceEpoch(infos.start))
currentWeek = time.strftime(
"%W", time.gmtime(StatsDateLib.getSecondsSinceEpoch(infos.start)))
fileName = paths.STATSCSVFILES
if infos.span == "daily":
fileName = fileName + "/" + _(
"daily/") + infos.fileType + "/%s/%s/%s/%s.csv" % (
machinesStr, currentYear, currentMonth, currentDay)
elif infos.span == "weekly":
fileName = fileName + "/" + _(
"weekly/") + infos.fileType + "/%s/%s/%s.csv" % (
machinesStr, currentYear, currentWeek)
elif infos.span == "monthly":
fileName = fileName + "/" + _(
"monthly/") + infos.fileType + "/%s/%s/%s.csv" % (
machinesStr, currentYear, currentMonth)
elif infos.span == "yearly":
fileName = fileName + "/" + _(
"yearly/") + infos.fileType + "/%s/%s.csv" % (machinesStr,
currentYear)
StatsDateLib.setLanguage(LanguageTools.getMainApplicationLanguage())
return fileName
def getTimeSinceLastUpdate(self, currentTimeInIsoFormat = "" ):
"""
@summary : returns the number of seconds between the last update
and the currentTime
@param currentTimeInIsoFormat: Current time specified in the ISO
format
@return : the number of seconds between the last update
and the currentTime
"""
timeBetweenUpdates = 0
if currentTimeInIsoFormat == "":
currentTimeInIsoFormat = StatsDateLib.getCurrentTimeInIsoformat()
currentTimeInSSEFormat = StatsDateLib.getSecondsSinceEpoch( currentTimeInIsoFormat )
lastUpdateInSSEFormat = StatsDateLib.getSecondsSinceEpoch( self.getTimeOfLastUpdateInLogs() )
if currentTimeInSSEFormat > lastUpdateInSSEFormat :
timeBetweenUpdates = currentTimeInSSEFormat - lastUpdateInSSEFormat
return timeBetweenUpdates
def getThreeClosestDatabasesBackups(infos):
"""
@summary : Returns the three databases backups
that are the closest to the startTime
asked for the database recollection.
@param infos :
@return: the three databases backups
that are the closest to the startTime
asked for the database recollection.
"""
closestFiles = []
differenceFileTuples = []
files = os.listdir(StatsPaths.STATSDB + 'databasesTimeOfUpdatesBackups/')
startTimeInEpochformat = StatsDateLib.getSecondsSinceEpoch(
infos.databasesRecollectionStartTime)
for file in files:
#try:
fileDateInIsoFormat = "%s %s" % (str(file).split("_")[0],
str(file).split("_")[1])
tupleToAdd = (abs(
StatsDateLib.getSecondsSinceEpoch(fileDateInIsoFormat) -
startTimeInEpochformat), file)
differenceFileTuples.append(tupleToAdd)
for tuple in differenceFileTuples[:3]:
closestFiles.append(tuple[1])
return closestFiles
def addOptions( parser ):
"""
This method is used to add all available options to the option parser.
"""
parser.add_option("-c", "--clients", action="store", type="string", dest="clients", default=_("ALL"),
help=_("Clients' names") )
parser.add_option( "--copy", action="store_true", dest = "copy", default=False, help=_("Create a copy file for the generated image.") )
parser.add_option("-d", "--daily", action="store_true", dest = "daily", default=False, help=_("Create daily graph(s).") )
parser.add_option( "--date", action="store", type="string", dest="date", default=StatsDateLib.getIsoFromEpoch( time.time() ), help=_("Decide end time of graphics. Usefull for testing.") )
parser.add_option("-f", "--fileType", action="store", type="string", dest="fileType", default='tx', help=_("Type of log files wanted."))
parser.add_option( "--fixedPrevious", action="store_true", dest="fixedPrevious", default=False, help=_("Do not use floating weeks|days|months|years. Use previous fixed interval found."))
parser.add_option( "--fixedCurrent", action="store_true", dest="fixedCurrent", default=False, help=_("Do not use floating weeks|days|months|years. Use current fixed interval found."))
parser.add_option( "--havingRun", action="store_true", dest="havingRun", default=False, help=_("Do not use only the currently running client/sources. Use all that have run between graphic(s) start and end instead."))
parser.add_option("-i", "--individual", action="store_true", dest = "individual", default=False, help=_("Dont combine data from specified machines. Create graphs for every machine independently") )
parser.add_option( "-l", "--language", action="store", type="string", dest="outputLanguage", default="", help = _("Language in which you want the graphic(s) details to be printed in.." ) )
parser.add_option("-m", "--monthly", action="store_true", dest = "monthly", default=False, help=_("Create monthly graph(s).") )
parser.add_option( "--machines", action="store", type="string", dest="machines", default=LOCAL_MACHINE, help = _("Machines for wich you want to collect data." ) )
parser.add_option("-s", "--span", action="store",type ="int", dest = "timespan", default=None, help=_("timespan( in hours) of the graphic.") )
parser.add_option("-t", "--types", type="string", dest="types", default=_("All"),help=_("Types of data to look for.") )
parser.add_option("--totals", action="store_true", dest = "totals", default=False, help=_("Create graphics based on the totals of all the values found for all specified clients or for a specific file type( tx, rx )."))
parser.add_option("--turnOffLogging", action="store_true", dest = "turnOffLogging", default=False, help=_("Turn off the logger"))
parser.add_option("-w", "--weekly", action="store_true", dest = "weekly", default=False, help=_("Create weekly graph(s)."))
parser.add_option("-y", "--yearly", action="store_true", dest = "yearly", default=False, help=_("Create yearly graph(s)."))
def getDirListToKeep(daysToKeep=21):
"""
@summary : Gets the list of directories to keep. Based on daysToKeep parameter.
@param : Number of past days to keep. Specified in daysToKeep.
@return : List of directories to keep.
"""
dirlist = []
secondsSinceEpoch = time.time()
for i in range(daysToKeep):
dirlist.append(
StatsDateLib.getIsoFromEpoch(secondsSinceEpoch -
(i * 60 * 60 *
24)).split()[0].replace('-', ''))
return dirlist
def getStartAndEndTimeForPickleRecollection():
"""
@summary : Gets the start time and the endTime
of the pickle recollection from the
user's input.
@return : Returns the startTime and endTime.
"""
startTime = raw_input(
"Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss) : "
)
while not StatsDateLib.isValidIsoDate(startTime):
print "Error. The entered date must be of the iso format."
startTime = raw_input(
"Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss) : "
)
endTime = raw_input(
"Enter the endTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
while (str(endTime).lower() != "now"
and not StatsDateLib.isValidIsoDate(endTime) and
(StatsDateLib.isValidIsoDate(endTime) and endTime <= startTime)):
if StatsDateLib.isValidIsoDate(endTime) and endTime <= startTime:
print "Error. End time must be after startTime( %s ). "
elif StatsDateLib.isValidIsoDate(endTime):
print "Error. The entered date must be of the iso format."
endTime = raw_input(
"Enter the endTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
if endTime == "now":
endTime = StatsDateLib.getIsoFromEpoch(time.time())
return startTime, endTime
def getGraphicProducerFromParserOptions( parser ):
"""
This method parses the argv received when the program was called
It takes the params wich have been passed by the user and sets them
in the corresponding fields of the infos variable.
If errors are encountered in parameters used, it will immediatly terminate
the application.
"""
graphicType = _("other")
mergerType = ""
( options, args )= parser.parse_args()
timespan = options.timespan
machines = options.machines.replace( ' ','').split(',')
clientNames = options.clients.replace( ' ','' ).split(',')
types = options.types.replace( ' ', '').split(',')
date = options.date.replace('"','').replace("'",'')
fileType = options.fileType.replace("'",'')
havingRun = options.havingRun
individual = options.individual
totals = options.totals
daily = options.daily
weekly = options.weekly
monthly = options.monthly
yearly = options.yearly
fixedCurrent = options.fixedCurrent
fixedPrevious = options.fixedPrevious
copy = options.copy
turnOffLogging = options.turnOffLogging
outputLanguage = options.outputLanguage
if outputLanguage == "":
outputLanguage = LanguageTools.getMainApplicationLanguage()
else :
if outputLanguage not in LanguageTools.getSupportedLanguages():
print _("Error. The specified language is not currently supported by this application.")
print _("Please specify one of the following languages %s or use the default value()" %( str( LanguageTools.getSupportedLanguages() ).replace("[","").replace("]",""), LanguageTools.getMainApplicationLanguage() ) )
print _("Program terminated.")
sys.exit()
counter = 0
specialParameters = [daily, monthly, weekly, yearly]
for specialParameter in specialParameters:
if specialParameter:
counter = counter + 1
if counter > 1 :
print _("Error. Only one of the daily, weekly and yearly options can be use at a time ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif counter == 1 and timespan != None :
print _("Error. When using the daily, the weekly or the yearly options timespan cannot be specified. " )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
elif counter == 0:
if fixedPrevious or fixedCurrent:
print _("Error. When using one of the fixed options, please use either the -d -m -w or -y options. " )
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if copy :
if daily or not( weekly or monthly or yearly ):
print _("Error. Copying can only be used with the -m -w or -y options. ")
print _("Use -h for help.")
print _("Program terminated.")
if counter == 0 and timespan == None :
timespan = 12
if fixedPrevious and fixedCurrent:
print _("Error. Please use only one of the fixed options,either fixedPrevious or fixedCurrent. ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if individual and totals:
print _("Error. Please use only one of the group options,either individual or totals. ")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
try: # Makes sure date is of valid format.
# Makes sure only one space is kept between date and hour.
t = time.strptime( date, '%Y-%m-%d %H:%M:%S' )
split = date.split()
date = "%s %s" %( split[0], split[1] )
except:
print _("Error. The date format must be YYYY-MM-DD HH:MM:SS")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
#Set graphic type based on parameters. Only one tpye is allowed at once based on previous validation.
if daily :
graphicType = _("daily")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24
elif weekly:
graphicType = _("weekly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 7
elif monthly:
graphicType = _("monthly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 30
elif yearly:
graphicType = _("yearly")
if fixedPrevious == False and fixedCurrent == False :
timespan = 24 * 365
start, end = StatsDateLib.getStartEndInIsoFormat(date, timespan, graphicType, fixedCurrent, fixedPrevious )
timespan = int( StatsDateLib.getSecondsSinceEpoch( end ) - StatsDateLib.getSecondsSinceEpoch( start ) ) / 3600
#print "timespan %s" %timespan
try:
if int( timespan ) < 1 :
raise
except:
print _("Error. The timespan value needs to be an integer one above 0.")
print _("Use -h for help.")
print _("Program terminated.")
sys.exit()
if fileType != "tx" and fileType != "rx":
print _("Error. File type must be either tx or rx.")
print "Specified file type was : ", fileType
print _("Multiple types are not accepted.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if havingRun == True and clientNames[0] != _("ALL"):
print _("Error. Cannot use the havingRun option while specifying client/source names.")
print _("To use havingRun, do not use -c|--client option.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if clientNames[0] == _("ALL"):
# Get all of the client/sources that have run between graph's start and end.
if totals == True or havingRun == True :
#print start, end, machines
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNamesHavingRunDuringPeriod( start, end, machines,None, havingrunOnAllMachines = True )
mergerType = _("totalForMachine")
else:#Build graphs only for currently runningclient/sources.
rxNames, txNames = GeneralStatsLibraryMethods.getRxTxNames( LOCAL_MACHINE, machines[0] )
mergerType = _("group")
if fileType == _("tx"):
clientNames = txNames
#print clientNames
else:
clientNames = rxNames
else:
if totals == True :
mergerType = _("regular")
#--------------------------------------------------------------------- try :
if fileType == _("tx"):
validTypes = [ _("latency"), _("bytecount"), _("errors"), _("filesOverMaxLatency"), _("filecount") ]
if types[0] == _("All") :
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("%s is not a valid type" %t)
else:
validTypes = [ _("bytecount"), _("errors"), _("filecount") ]
if types[0] == _("All"):
types = validTypes
else :
for t in types :
if t not in validTypes:
raise Exception("")
#------------------------------------------------------------------- except:
#----------------------------------------------------------- print types
# print _("Error. With %s fileType, possible data types values are : %s.") %( fileType, validTypes )
#---- print _("For multiple types use this syntax : -t 'type1','type2'")
#-------------------------------- print _("Use -h for additional help.")
#---------------------------------------- print _("Program terminated.")
#------------------------------------------------------------ sys.exit()
if individual != True :
combinedMachineName = ""
for machine in machines:
combinedMachineName = combinedMachineName + machine
machines = [ combinedMachineName ]
if len(clientNames) <1:
print _("Error. No client/sources were found that matched the specified parameters") %( fileType, validTypes )
print _("Verify parameters used, especially the machines parameter.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
if len(clientNames) <1:
print _("Error. No client/sources were found that matched the specified parameters")
print _("Verify parameters used, especially the machines parameter.")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
elif len(clientNames) == 1 and totals == True:
print _("Error. Cannot use totals option with only one client/source name.")
print _("Either remove --total option or use more than one client/source..")
print _("Use -h for additional help.")
print _("Program terminated.")
sys.exit()
end = StatsDateLib.getIsoWithRoundedHours( end )
graphicsProducer = RRDGraphicProducer( startTime = start, endTime = end, graphicType = graphicType, clientNames = clientNames, types = types, timespan = timespan, machines = machines, fileType = fileType,\
totals = totals, copy = copy, mergerType = mergerType,turnOffLogging = turnOffLogging, inputLanguage = LanguageTools.getMainApplicationLanguage() , outputLanguage = outputLanguage )
return graphicsProducer
def getStartAndEndTimeForDatabaseRecollection(infos):
"""
@summary : Gets the start time and the endTime
of the pickle recollection from the
user's input.
@param infos : Previously gathered infos.
@note : If pickles are to be recollected,
infos must contain the pickles
recollection start time and end time.
@return : Returns the startTime and endTime.
"""
if infos.pickles == True:
isCertainAboutStartTime = False
#************************startTime section*********
while isCertainAboutStartTime == False:
startTime = raw_input(
"Enter the startTime of the dataBase recollection ( yyyy-mm-dd hh:mm:ss ) : "
)
while not StatsDateLib.isValidIsoDate(startTime):
if not StatsDateLib.isValidIsoDate(startTime):
print "Error. The entered date must be of the iso format."
startTime = raw_input(
"Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss ) : "
)
if (StatsDateLib.isValidIsoDate(startTime)
and startTime > infos.picklesRecollectionStartTime):
print "Warning : StartTime of database recollection ( %s ) is after startTime of pickleRecollection( %s ) " % (
startTime, infos.picklesRecollectionStartTime)
isCertainAnswer = raw_input(
"Are you sure you want to keep this date ? ( y or n ) : ")
while (str(isCertainAnswer).lower() != 'y'
and str(isCertainAnswer).lower() != 'n'):
print "Error.Answer needs to be either y or n."
isCertainAnswer = raw_input(
"Are you sure you want to keep this date ? ( y or n ) : "
)
if str(isCertainAnswer).lower() == 'y':
isCertainAboutStartTime = True
else:
print "A new startTime will be required."
else:
isCertainAboutStartTime = True
#************************endTime section*********
isCertainAboutEndTime = False
while isCertainAboutEndTime == False:
endTime = raw_input(
"Enter the endTime of the dataBase recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
while (not StatsDateLib.isValidIsoDate(endTime)
and str(endTime).lower() != "now"):
if not StatsDateLib.isValidIsoDate(endTime):
print "Error. The entered date must be of the iso format or now."
endTime = raw_input(
"Enter the endTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
if (endTime != "now" and StatsDateLib.isValidIsoDate(endTime)
and endTime < infos.picklesRecollectionEndTime):
print "Warning : endTime of database recollection ( %s ) is before the endTime of pickleRecollection( %s ) " % (
startTime, infos.picklesRecollectionStartTime)
isCertainAnswer = raw_input(
"Are you sure you want to keep this date ? ( y or n ) : ")
while (str(isCertainAnswer).lower() != 'y'
and str(isCertainAnswer).lower() != 'n'):
print "Error.Answer needs to be either y or n."
isCertainAnswer = raw_input(
"Are you sure you want to keep this date ? ( y or n ) : "
)
if str(isCertainAnswer).lower() == 'y':
isCertainAboutEndTime = True
else:
print "A new endTime will be required."
else:
isCertainAboutEndTime = True
if endTime == "now":
endTime = StatsDateLib.getIsoFromEpoch(time.time())
isCertainAboutEndTime = True
else:
startTime = raw_input(
"Enter the startTime of the dataBase recollection ( yyyy-mm-dd hh:mm:ss ) : "
)
while not StatsDateLib.isValidIsoDate(startTime):
if not StatsDateLib.isValidIsoDate(startTime):
print "Error. The entered date must be of the iso format."
startTime = raw_input(
"Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss) : "
)
endTime = raw_input(
"Enter the endTime of the dataBase recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
while (str(endTime) != "now"
and not StatsDateLib.isValidIsoDate(endTime)):
if not StatsDateLib.isValidIsoDate(endTime):
print "Error. The entered date must be of the iso format."
endTime = raw_input(
"Enter the startTime of the pickle recollection ( yyyy-mm-dd hh:mm:ss or 'now' for current time ) : "
)
if endTime == "now":
endTime = StatsDateLib.getIsoFromEpoch(time.time())
return startTime, endTime
def printWebPage(self, rxNames, txNames):
"""
@summary : Prints out a daily web page
with the content based on
the specified parameters.
@param rxNames: List of rx for which to display the graphics
@param txNames: List of rx for which to display the graphics
@precondition: global _ translator must be set prior to calling this function.
@note : Only links to available graphics will be
displayed.
"""
global _
StatsDateLib.setLanguage(self.displayedLanguage)
rxNamesArray = rxNames.keys()
txNamesArray = txNames.keys()
rxNamesArray.sort()
txNamesArray.sort()
#Redirect output towards html page to generate.
if not os.path.isdir(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML):
os.makedirs(self.pathsTowardsOutputFiles.STATSWEBPAGESHTML)
fileHandle = open(
self.pathsTowardsOutputFiles.STATSWEBPAGESHTML +
"dailyGraphs_%s.html" % self.displayedLanguage, 'w')
fileHandle.write("""
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
<link rel="stylesheet" href="../scripts/js_%s/windowfiles/dhtmlwindow.css" type="text/css" />
<script type="text/javascript" src="../scripts/js_%s/windowfiles/dhtmlwindow.js">
This is left here to give credit to the original
creators of the dhtml script used for the group pop ups:
/***********************************************
* DHTML Window Widget- Dynamic Drive (www.dynamicdrive.com)
* This notice must stay intact for legal use.
* Visit http://www.dynamicdrive.com/ for full source code
***********************************************/
</script>
""" % (self.displayedLanguage, self.displayedLanguage) + """
<script type="text/javascript">
var descriptionWindow=dhtmlwindow.open("description", "inline", "description", "Description", "width=900px,height=120px,left=150px,top=10px,resize=1,scrolling=0", "recal")
descriptionWindow.hide()
</script>
<head>
<title> PX Graphics </title>
</head>
<script>
counter =0;
function wopen(url, name, w, h){
// This function was taken on www.boutell.com
w += 32;
h += 96;
counter +=1;
var win = window.open(url,
counter,
'width=' + w + ', height=' + h + ', ' +
'location=no, menubar=no, ' +
'status=no, toolbar=no, scrollbars=no, resizable=no');
win.resizeTo(w, h);
win.focus();
}
</script>
<script>
function showSourceHelpPage(){
var sourceHelpPage = dhtmlwindow.open("sourceHelpPage", "iframe", "helpPages/source_%s.html", "Definition of 'source'", "width=875px,height=100px,resize=1,scrolling=1,center=1", "recal")
sourceHelpPage.moveTo("middle", "middle");
}""" % (self.displayedLanguage) + """
function showClientHelpPage(){
var clientHelpPage = dhtmlwindow.open("client", "iframe", "helpPages/client_%s.html", "Definition of 'client'", "width=875px,height=150px,resize=1,scrolling=1,center=1", "recal")
.moveTo("middle", "middle");
}""" % (self.displayedLanguage) + """
</script>
<STYLE>
<!--
A{text-decoration:none}
-->
</STYLE>
<style type="text/css">
div.left { float: left; }
div.right {float: right; }
</style>
<style type="text/css">
div.tableContainer {
width: 95%; /* table width will be 99% of this*/
height: 275px; /* must be greater than tbody*/
overflow: auto;
margin: 0 auto;
}
table {
width: 99%; /*100% of container produces horiz. scroll in Mozilla*/
border: none;
background-color: #f7f7f7;
table-layout: fixed;
}
table>tbody { /* child selector syntax which IE6 and older do not support*/
overflow: auto;
height: 225px;
overflow-x: hidden;
}
thead tr {
position:relative;
}
thead td, thead th {
text-align: center;
font-size: 14px;
background-color:"#006699";
color: steelblue;
font-weight: bold;
border-top: solid 1px #d8d8d8;
}
td {
color: #000;
padding-right: 2px;
font-size: 12px;
text-align: left;
border-bottom: solid 1px #d8d8d8;
border-left: solid 1px #d8d8d8;
}
tfoot td {
text-align: center;
font-size: 11px;
font-weight: bold;
background-color: papayawhip;
color: steelblue;
border-top: solid 2px slategray;
}
td:last-child {padding-right: 20px;} /*prevent Mozilla scrollbar from hiding cell content*/
</style>
<body text="#000000" link="#FFFFFF" vlink="000000" bgcolor="#FFF4E5" >
<h2>""" + _("Daily graphics for RX sources from MetPx.") +
""" <font size = "2">""" + _("*updated hourly") +
"""</font></h2>
<div class="tableContainer">
<table>
<thead>
<tr>
<td bgcolor="#006699">
<div class = "rxTableEntry">
<font color = "white">
<center>
Sources
<br>
<a target ="popup" href="#" onClick="showSourceHelpPage(); return false;">
?
</a>
</center>
</font>
</div>
</td>
<td bgcolor="#006699">
<font color = "white">""" +
_("List of available daily graphics.") +
"""</font>
</td>
</tr>
</thead>
<tbody>
""")
for rxName in rxNamesArray:
if rxNames[rxName] == "":
fileHandle.write("""<tr> <td bgcolor="#99FF99"> %s</td> """ %
(rxName))
fileHandle.write("""<td bgcolor="#66CCFF"> """ + _("Days") +
""" : """)
else:
machineName = self.getMachineNameFromDescription(
rxNames[rxName])
fileHandle.write(
"""<tr> <td bgcolor="#99FF99"><div class="left"> %s</div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """
% (rxName, rxNames[rxName].replace("'", "").replace(
'"', '').replace(",", ", "), machineName))
fileHandle.write("""<td bgcolor="#66CCFF"> """ + _("Days") +
""" : """)
for day in self.days:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(
day)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH,
self.filesLanguage)
file = file = self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _(
"daily/rx/%s/") % (rxName) + str(currentYear) + "/" + str(
currentMonth) + "/" + str(currentDay) + ".png"
webLink = _("archives/daily/rx/%s/") % (rxName) + str(
currentYear) + "/" + str(currentMonth) + "/" + str(
currentDay) + ".png"
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH,
self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write("""<a target ="%s" href="%s">""" %
(rxName, webLink) +
"%s" % StatsDateLib.getDayOfTheWeek(day) +
""" </a>""")
else:
pass
fileHandle.write("""</td></tr>""")
fileHandle.write("""
</tbody>
</table>
</div>
<h2>""" + _("Daily graphics for TX clients from MetPx.") +
""" <font size = "2">""" + _("*updated hourly") +
"""</font></h2>
<div class="tableContainer">
<table>
<thead>
<tr>
<td bgcolor="#006699">
<div class = "txTableEntry">
<font color = "white">
<center>
Clients
<br>
<a target ="popup" href="#" onClick="showClientHelpPage(); return false;">
?
</a>
<center>
</font>
</div>
</td>
<td bgcolor="#006699">
<div class = "txTableEntry">
<font color = "white">""" +
_("List of available daily graphics.") + """</font>
</div>
</td>
</tr>
</thead>
<tbody>
""")
for txName in txNamesArray:
if txNames[txName] == "":
fileHandle.write("""<tr> <td bgcolor="#99FF99"> %s</td> """ %
(txName))
fileHandle.write(
"""<td bgcolor="#66CCFF"><div class = "txTableEntry"> """
+ _("Days") + """ : """)
else:
machineName = self.getMachineNameFromDescription(
txNames[txName])
fileHandle.write(
"""<tr> <td bgcolor="#99FF99"><div class="left"> %s </div><div class="right"><a href="#" onClick="descriptionWindow.load('inline', '%s', 'Description');descriptionWindow.show(); return false"><font color="black">?</font></a></div><br>(%s)</td> """
% (txName, txNames[txName].replace("'", "").replace(
'"', '').replace(",", ", "), machineName))
fileHandle.write("""<td bgcolor="#66CCFF"> """ + _("Days") +
""" : """)
for day in self.days:
currentYear, currentMonth, currentDay = StatsDateLib.getYearMonthDayInStrfTime(
day)
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH,
self.filesLanguage)
file = self.pathsTowardsGraphics.STATSGRAPHSARCHIVES + _(
"/daily/tx/%s/") % (txName) + str(currentYear) + "/" + str(
currentMonth) + "/" + str(currentDay) + ".png"
webLink = _("archives/daily/tx/%s/") % (txName) + str(
currentYear) + "/" + str(currentMonth) + "/" + str(
currentDay) + ".png"
_ = self.getTranslatorForModule(CURRENT_MODULE_ABS_PATH,
self.displayedLanguage)
if os.path.isfile(file):
fileHandle.write("""<a target ="%s" href="%s">""" %
(rxName, webLink) +
"%s" % StatsDateLib.getDayOfTheWeek(day) +
""" </a>""")
else:
pass
fileHandle.write("</td></tr>")
fileHandle.write("""
</tbody>
</table>
</body>
</html>
""")
fileHandle.close()
def getParametersFromForm(self, form):
"""
@summary: Initialises the queryParameters
based on the form received as
parameter.
@note : Absent parameters will be set to default values( [] or '' )
and will NOT raise exceptions. Use the searchForParameterErrors
function to search for errors
"""
global _
#print form
image = None #No image was produced yet
#Every param is received in an array, use [0] to get first item, nothing for array.
try:
querier = form["querier"].replace("'", "").replace('"', '')
except:
querier = ''
try:
plotter = form["plotter"].replace("'", "").replace('"', '')
except:
plotter = ''
try:
fileTypes = form["fileType"].replace("'", "").replace('"', '')
except:
fileTypes = ''
try:
sourLients = form["sourLients"].split(',')
except:
sourLients = []
try:
groupName = form["groupName"].replace("'", "").replace('"', '')
except:
groupName = ''
try:
machines = form["machines"].split(',')
except:
machines = []
if groupName != '' and (sourLients == [] or sourLients == ['']):
configParameters = StatsConfigParameters()
configParameters.getAllParameters()
if groupName in configParameters.groupParameters.groups:
if configParameters.groupParameters.groupFileTypes[
groupName] == fileTypes and configParameters.groupParameters.groupsMachines[
groupName] == machines:
sourLients = configParameters.groupParameters.groupsMembers[
groupName]
try:
combine = form["combineSourlients"].replace(",",
"").replace('"', '')
if combine == 'false' or combine == 'False':
combine = False
elif combine == 'true' or combine == 'True':
combine = True
else:
raise
except:
combine = False
try:
endTime = form["endTime"].replace("'", "").replace('"', '')
hour = endTime.split(" ")[1]
splitDate = endTime.split(" ")[0].split('-')
endTime = "%s" % (splitDate[2] + '-' + splitDate[1] + '-' +
splitDate[0] + " " + hour)
if _("current") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromCurrentDay(
endTime)
elif _("previous") in str(form["fixedSpan"]).lower():
start, endTime = StatsDateLib.getStartEndFromPreviousDay(
endTime)
except:
endTime = ''
try:
products = form["products"].split(',')
if products == [""]:
raise
except:
products = ["*"]
try:
statsTypes = form["statsTypes"].split(',')
except:
statsTypes = []
#statsTypes = translateStatsTypes( statsTypes )
try:
span = form["span"].replace("'", "").replace('"', '')
if str(span).replace(' ', '') == '':
raise
span = int(span)
except:
span = 24
try:
language = form["lang"].replace("'", "").replace('"', '')
except:
language = ""
sourLients = GeneralStatsLibraryMethods.filterClientsNamesUsingWilcardFilters(
endTime, span, sourLients, machines, [fileTypes])
self.queryParameters = GnuQueryBroker._QueryParameters(
fileTypes, sourLients, groupName, machines, combine, endTime,
products, statsTypes, span, language)
self.replyParameters = GnuQueryBroker._ReplyParameters(
querier, plotter, image, fileTypes, sourLients, groupName,
machines, combine, endTime, products, statsTypes, span, '',
language)