def getSavedList( self, user, clients ): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try : self.savedFileList = CpickleWrapper.load( directory + fileName ) if self.savedFileLis == None : self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def loadAccessFile(self): """ @summary: Loads the accessFile into the accessDictionary. """ self.accessDictionary = CpickleWrapper.load( self.accessFile )
def getSavedList(self, user, clients): """ @summary : Returns the checksum of the files contained in the saved list. """ self.savedFileList = {} statsPaths = StatsPaths() statsPaths.setPaths() directory = statsPaths.STATSDATA + "fileAccessVersions/" combinedName = "" for client in clients: combinedName = combinedName + client fileName = combinedName + "_" + user try: self.savedFileList = CpickleWrapper.load(directory + fileName) if self.savedFileLis == None: self.savedFileList = {} except: # if file does not exist pass return self.savedFileList
def loadAccessFile(self): """ @summary: Loads the accessFile into the accessDictionary. """ self.accessDictionary = CpickleWrapper.load(self.accessFile)
def printPickle(pickle, outputFile=""): """ Print content of a pickle file containing a FileStatscollector instance on the desired output. Default output is the console screen. File can be specified to make reading easier. """ if outputFile != "": fileHandle = open(outputFile, 'w') old_stdout = sys.stdout sys.stdout = fileHandle statsCollection = CpickleWrapper.load(pickle) print _("Pickle used : %s") % pickle print _("\n\nFiles used : %s") % statsCollection.files print _("Starting date: %s") % statsCollection.startTime print _("Interval: %s") % statsCollection.interval print _("End time : %s") % statsCollection.endTime print _("nbEntries : %s") % statsCollection.nbEntries for j in range(statsCollection.nbEntries): print _("\nEntry's interval : %s - %s ") % ( statsCollection.fileEntries[j].startTime, statsCollection.fileEntries[j].endTime) print _("Files : ") print statsCollection.fileEntries[j].files print _("Products : ") print statsCollection.fileEntries[j].values.productTypes print _("Values :") print statsCollection.fileEntries[j].values.dictionary print _("Means :") print statsCollection.fileEntries[j].means print _("Medians") print statsCollection.fileEntries[j].medians print _("Minimums") print statsCollection.fileEntries[j].minimums print _("Maximums") print statsCollection.fileEntries[j].maximums print _("Time where max occured :") print statsCollection.fileEntries[j].timesWhereMaxOccured print _("Total") print statsCollection.fileEntries[j].totals print _("Files over maximum latency") print statsCollection.fileEntries[j].filesOverMaxLatency if outputFile != "": fileHandle.close() sys.stdout = old_stdout #resets standard output
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\ endTime = "2006-07-31 19:00:00", client = "satnet",\ machine = "pdsPM", fileType = "tx" ): """ @summary : This method merges entire hourly pickles files together. @None : This does not support merging part of the data of pickles. """ if logger != None : logger.debug( _("Call to mergeHourlyPickles received.") ) logging = True else: logging = False pickles = [] entries = {} width = StatsDateLib.getSecondsSinceEpoch( endTime ) - StatsDateLib.getSecondsSinceEpoch( startTime ) startTime = StatsDateLib.getIsoWithRoundedHours( startTime ) seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime( startTime = startTime , width=width, interval=60*StatsDateLib.MINUTE )[:-1]) for seperator in seperators : pickles.append( StatsPickler.buildThisHoursFileName( client = client, offset = 0, currentTime = seperator, machine = machine, fileType = fileType ) ) startingNumberOfEntries = 0 #print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) for pickle in pickles : if os.path.isfile( pickle ) : tempCollection = CpickleWrapper.load( pickle ) if tempCollection != None : for i in xrange( len( tempCollection.fileEntries ) ): entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i] startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries ) else: sys.exit() else: emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60, entries = {} ) for i in xrange( 60 ): entries[i + startingNumberOfEntries ] = emptyEntries [i] startingNumberOfEntries = startingNumberOfEntries + 60 #print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) statsCollection = FileStatsCollector( startTime = startTime , endTime = endTime, interval = StatsDateLib.MINUTE, totalWidth = width, fileEntries = entries,fileType= fileType, logger = logger, logging = logging ) return statsCollection
def printPickle( pickle, outputFile = "" ): """ Print content of a pickle file containing a FileStatscollector instance on the desired output. Default output is the console screen. File can be specified to make reading easier. """ if outputFile != "": fileHandle = open( outputFile , 'w' ) old_stdout = sys.stdout sys.stdout = fileHandle statsCollection = CpickleWrapper.load( pickle ) print _("Pickle used : %s" )%pickle print _("\n\nFiles used : %s" ) %statsCollection.files print _("Starting date: %s" ) % statsCollection.startTime print _("Interval: %s" ) %statsCollection.interval print _("End time : %s" ) %statsCollection.endTime print _("nbEntries : %s" ) %statsCollection.nbEntries for j in range( statsCollection.nbEntries ): print _("\nEntry's interval : %s - %s " ) %( statsCollection.fileEntries[j].startTime, statsCollection.fileEntries[j].endTime ) print _("Files : " ) print statsCollection.fileEntries[j].files print _("Products : " ) print statsCollection.fileEntries[j].values.productTypes print _("Values :" ) print statsCollection.fileEntries[j].values.dictionary print _("Means :" ) print statsCollection.fileEntries[j].means print _("Medians" ) print statsCollection.fileEntries[j].medians print _("Minimums" ) print statsCollection.fileEntries[j].minimums print _("Maximums" ) print statsCollection.fileEntries[j].maximums print _("Time where max occured :" ) print statsCollection.fileEntries[j].timesWhereMaxOccured print _("Total" ) print statsCollection.fileEntries[j].totals print _("Files over maximum latency" ) print statsCollection.fileEntries[j].filesOverMaxLatency if outputFile != "": fileHandle.close() sys.stdout = old_stdout #resets standard output
def changeInUpdateFrenquencyFoundDuringTimespan( self, startTime, endTime ): """ @summary : Searchs whether or not there was a change during the specified timespan. @param statTime : Start time in the iso format of the time span to survey. @param endTime : End time in the iso format of the time span to survey/ @return : True or false whether there was a change or not, plus the original frequency and the new frequency. """ changeWasMade = False paths = StatsPaths() paths.setPaths() updatesDoneDuringTimespan = self.__getAutomaticUpdatesDoneDuringTimeSpan( startTime, endTime ) updatesDoneDuringTimespan.sort() if updatesDoneDuringTimespan != []: fileName = paths.STATSTEMPAUTUPDTLOGS + str(updatesDoneDuringTimespan[0]).replace( " ", "_" ) originalUpdateFrequency = CpickleWrapper.load(fileName) newUpdateFrequency = originalUpdateFrequency for update in updatesDoneDuringTimespan: fileName = paths.STATSTEMPAUTUPDTLOGS + str(update).replace( " ", "_" ) newUpdateFrequency = CpickleWrapper.load(fileName) if newUpdateFrequency != originalUpdateFrequency: changeWasMade = True break return changeWasMade, originalUpdateFrequency, newUpdateFrequency
def getMachineParametersFromPreviousCall() : """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile( paths.STATSPREVIOUSMACHINEPARAMS ): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS ) return previousMachineParams
def previousUpdateFrequency(self): """ @summary : Finds and returns the frequency of the previous update. @return : The freqency of the previous update. """ paths = StatsPaths() paths.setPaths() lastUpdate = self.getTimeOfLastUpdateInLogs() fileName = paths.STATSTEMPAUTUPDTLOGS + str(lastUpdate).replace( " ", "_" ) lastUpdateFrequency = CpickleWrapper.load(fileName) return lastUpdateFrequency
def getMachineParametersFromPreviousCall(): """ @summary: Gets the machine parameters that are saved in data/previousMachineParameters. @return: Returns the saved machine parameters. """ paths = StatsPaths() paths.setPaths() previousMachineParams = None if os.path.isfile(paths.STATSPREVIOUSMACHINEPARAMS): previousMachineParams = CpickleWrapper.load( paths.STATSPREVIOUSMACHINEPARAMS) return previousMachineParams
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None : logger.debug( _("Call to mergePickles received.") ) logging = True else: logging = False entryList = [] for pickle in pickleNames:#for every pickle we eneed to merge if os.path.isfile( pickle ): entryList.append( CpickleWrapper.load( pickle ) ) else:#Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch( currentTime ) + StatsDateLib.HOUR ) entryList.append( FileStatsCollector( startTime = currentTime, endTime = endTime,logger =logger, logging =logging ) ) if logger != None : logger.warning( _("Pickle named %s did not exist. Empty entry was used instead.") %pickle ) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid( entryList ) == True : for i in range ( 1 , len( entryList ) ): #add other entries for file in entryList[i].files : if file not in newFSC.files : newFSC.files.append( file ) for j in range( len( newFSC.fileEntries ) ) : # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes ) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files ) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times ) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[j].nbFiles + ( newFSC.fileEntries[ j ].nbFiles) for type in newFSC.statsTypes : newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket = 0 , finishingBucket = newFSC.nbEntries -1 ) else:#Did not merge pickles named. Pickle list was not valid." if logger != None : logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid.") %pickleNames ) logger.warning( _("Filled with empty entries instead.") %pickleNames ) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries = 60 , entries = {} ) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save( newFSC, mergedPickleName ) try: os.chmod( mergedPickleName, 0777 ) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def mergePicklesFromSameHour( logger = None , pickleNames = None, mergedPickleName = "",\ clientName = "" , combinedMachineName = "", currentTime = "",\ fileType = "tx" ): """ @summary: This methods receives a list of filenames referring to pickled FileStatsEntries. After the merger pickles get saved since they might be reused somewhere else. @precondition: Pickle should be of the same timespan and bucket width. If not no merging will occur. """ if logger != None: logger.debug(_("Call to mergePickles received.")) logging = True else: logging = False entryList = [] for pickle in pickleNames: #for every pickle we eneed to merge if os.path.isfile(pickle): entryList.append(CpickleWrapper.load(pickle)) else: #Use empty entry if there is no existing pickle of that name endTime = StatsDateLib.getIsoFromEpoch( StatsDateLib.getSecondsSinceEpoch(currentTime) + StatsDateLib.HOUR) entryList.append( FileStatsCollector(startTime=currentTime, endTime=endTime, logger=logger, logging=logging)) if logger != None: logger.warning( _("Pickle named %s did not exist. Empty entry was used instead." ) % pickle) #start off with a carbon copy of first pickle in list. newFSC = FileStatsCollector( files = entryList[0].files , statsTypes = entryList[0].statsTypes, startTime = entryList[0].startTime,\ endTime = entryList[0].endTime, interval=entryList[0].interval, totalWidth = entryList[0].totalWidth,\ firstFilledEntry = entryList[0].firstFilledEntry, lastFilledEntry = entryList[0].lastFilledEntry,\ maxLatency = entryList[0].maxLatency, fileEntries = entryList[0].fileEntries,logger = logger,\ logging = logging ) if PickleMerging.entryListIsValid(entryList) == True: for i in range(1, len(entryList)): #add other entries for file in entryList[i].files: if file not in newFSC.files: newFSC.files.append(file) for j in range(len(newFSC.fileEntries)): # add all entries newFSC.fileEntries[j].values.productTypes.extend( entryList[i].fileEntries[j].values.productTypes) newFSC.fileEntries[j].files.extend( entryList[i].fileEntries[j].files) newFSC.fileEntries[j].times.extend( entryList[i].fileEntries[j].times) newFSC.fileEntries[j].nbFiles = newFSC.fileEntries[ j].nbFiles + (newFSC.fileEntries[j].nbFiles) for type in newFSC.statsTypes: newFSC.fileEntries[j].values.dictionary[type].extend( entryList[i].fileEntries[j].values.dictionary[type] ) newFSC.fileEntries[j].values.rows = newFSC.fileEntries[ j].values.rows + entryList[i].fileEntries[j].values.rows newFSC = newFSC.setMinMaxMeanMedians( startingBucket=0, finishingBucket=newFSC.nbEntries - 1) else: #Did not merge pickles named. Pickle list was not valid." if logger != None: logger.warning( _("Did not merge pickles named : %s. Pickle list was not valid." ) % pickleNames) logger.warning( _("Filled with empty entries instead.") % pickleNames) newFSC.fileEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries=60, entries={}) #prevents us from having ro remerge file later on. temp = newFSC.logger del newFSC.logger CpickleWrapper.save(newFSC, mergedPickleName) try: os.chmod(mergedPickleName, 0777) except: pass #print "saved :%s" %mergedPickleName newFSC.logger = temp return newFSC
def mergePicklesFromDifferentHours( logger = None , startTime = "2006-07-31 13:00:00",\ endTime = "2006-07-31 19:00:00", client = "satnet",\ machine = "pdsPM", fileType = "tx" ): """ @summary : This method merges entire hourly pickles files together. @None : This does not support merging part of the data of pickles. """ if logger != None: logger.debug(_("Call to mergeHourlyPickles received.")) logging = True else: logging = False pickles = [] entries = {} width = StatsDateLib.getSecondsSinceEpoch( endTime) - StatsDateLib.getSecondsSinceEpoch(startTime) startTime = StatsDateLib.getIsoWithRoundedHours(startTime) seperators = [startTime] seperators.extend( StatsDateLib.getSeparatorsWithStartTime(startTime=startTime, width=width, interval=60 * StatsDateLib.MINUTE)[:-1]) for seperator in seperators: pickles.append( StatsPickler.buildThisHoursFileName(client=client, offset=0, currentTime=seperator, machine=machine, fileType=fileType)) startingNumberOfEntries = 0 #print "prior to loading and merging pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) for pickle in pickles: if os.path.isfile(pickle): tempCollection = CpickleWrapper.load(pickle) if tempCollection != None: for i in xrange(len(tempCollection.fileEntries)): entries[startingNumberOfEntries + i] = tempCollection.fileEntries[i] startingNumberOfEntries = startingNumberOfEntries + len( tempCollection.fileEntries) else: sys.exit() else: emptyEntries = PickleMerging.fillWithEmptyEntries( nbEmptyEntries=60, entries={}) for i in xrange(60): entries[i + startingNumberOfEntries] = emptyEntries[i] startingNumberOfEntries = startingNumberOfEntries + 60 #print "after the loading and merging og pickles : %s " %( StatsDateLib.getIsoFromEpoch( time.time() ) ) statsCollection = FileStatsCollector(startTime=startTime, endTime=endTime, interval=StatsDateLib.MINUTE, totalWidth=width, fileEntries=entries, fileType=fileType, logger=logger, logging=logging) return statsCollection