def _getMetadata(self, comment, metadata, lineNr, sourceCrontab): data = None for i in range(0, len(comment)): meta = comment[i] if meta.startswith(metadata + "="): _, value = meta.split("=") if value != "": return value try: idx = comment.index(metadata) if comment[idx + 1] != "=": data = comment[idx + 1] data = data.replace("=", "") else: data = comment[idx + 2] except ValueError: try: idx = comment.index(metadata + "=") data = comment[idx + 1] except ValueError: CCronLogger.info(metadata.title() + " metadata not found in line " + str(lineNr) + " in " + sourceCrontab + ".") CCronLogger.info("It will be replaced by default value.") return data
def chooseRandomSample(self,centers,weights): centersAndWeights=[] #spojim do jednoho pole for i in range(0,len(centers)): centersAndWeights.append([centers[i],weights[i]]) #vyberu vzorek if self._approximation: try: sample=random.sample(centersAndWeights,self._samplesize) except ValueError: CCronLogger.info("Sample larger than population. Using all data.") CCronLogger.infoToConsole("Sample larger than population. Using all data.") sample=centersAndWeights sample=sorted(sample,key=getKey) else: sample=centersAndWeights #roztrhnu do dvou poli centers=[] weights=[] for center, weight in sample: centers.append(center) weights.append(weight) return centers, weights
def checkEmptyCronJobs(cronList): newCronList=[] badCronList=[] for cron in cronList: if len(cron.cronJobList) > 0: newCronList.append(cron) else: CCronLogger.info("Following cron entry:"+str(cron)+" wont be analyzed or optimized, because it wont run in specified time window.") badCronList.append(cron) return newCronList,badCronList
def createStatistics(self,maxWeight,weights): std=round(statistics.pstdev(weights),1) msg="Standard deviation of weights in test intervals: "+str(std) CCronLogger.infoToConsole(msg) CCronLogger.info(msg) msg="Max weight in test interval: "+str(maxWeight) CCronLogger.infoToConsole(msg) CCronLogger.info(msg) return std
def checkPredictionDuration(options): try: duration=int(options["predictionduration"]) if duration < 1: CCronLogger.error("Invalid duration of prediction. Setting to default: 30.") duration=30 CCronLogger.debug("Duration of prediction: "+str(duration)+" days.") except ValueError: CCronLogger.error("Invalid duration of prediction. Setting to default: 30.") duration=30 except KeyError: CCronLogger.info("Duration of prediction directive not found. Setting to default: 30.") duration=30 return duration
def __init__(self, options): self.setOptions(options) self._crontabFolders = [] #pridat slozky s crontaby do seznamu #obecna slozka s crontaby if self.includeCrondFolder: self._crontabFolders.append(["/etc/cron.d/", False]) #uzivatelske slozky s crontaby if os.geteuid() == 0: for usercrontabdir in self.userCrontabDirs.split(":"): self._crontabFolders.append([usercrontabdir, True]) else: CCronLogger.info( "You have to run this tool as root to access user crontabs.") CCronLogger.infoToConsole( "You have to run this tool as root to access user crontabs.") #ostatni slozky s crontaby, ktere nejsou uzivatelske for othercrontabdir in self.otherCrontabDirs.split(":"): if othercrontabdir != "": self._crontabFolders.append([othercrontabdir, False]) #muj testovaci crontab #self._crontabFolders.append(["/home/petr/git/cronalyzer/data/crontabs/",True]) self._crontabs = [] for crontabFolder, isUser in self._crontabFolders: for crontab in glob.glob(os.path.join(crontabFolder, "*")): if not crontab.endswith("~") and os.path.isfile(crontab): self._crontabs.append([crontab, isUser]) CCronLogger.info("Added crontab: " + crontab + " to analysis.") CCronLogger.infoToConsole("Added crontab: " + crontab + " to analysis.") # + /etc/crontab if self.includeMainCrontab: self._crontabs.append(["/etc/crontab", False]) CCronLogger.info("Added crontab: /etc/crontab to analysis.") CCronLogger.infoToConsole( "Added crontab: /etc/crontab to analysis.") #list s rozparsovanymi crony self._crons = []
def setOptions(self,options): try: self._outputDir=options["outputdir"] CCronLogger.debug("Output directory: "+self._outputDir) except KeyError: CCronLogger.info("Output directory directive not found. Setting to default: working directory.") self._outputDir="./" try: self._graphName=options["graphname"] CCronLogger.debug("Graph name: "+self._graphName) except KeyError: CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.") self._graphName="graph.pdf" try: self._weightGraphName=options["weightgraphname"] CCronLogger.debug("Weight graph name: "+self._weightGraphName) except KeyError: CCronLogger.info("Weight graph name directive not found. Setting to default: weightGraph.pdf.") self._weightGraphName="weightGraph.pdf"
def main(): #pokud neni zadan skript, zkusi se doplnit z aktualniho adresare if len(sys.argv[1:]) == 0: configFile = "config.conf" else: configFile = readArguments(sys.argv[1:]) testConfigFile(configFile) configParser = CConfigParser(configFile) options = configParser.parseConfig() #nacteni a vytvoreni slozky try: outputDir = options["outputdir"] except KeyError: outputDir = "." options["outputdir"] = repairOutputDir(outputDir) createOutputDir(outputDir) logFile = getLogFile(options) checkLogFile(logFile) try: logLevel = options["loglevel"] except KeyError: logFile = "info" try: removeLog = options["removelogonstart"] except KeyError: removeLog = False try: onlyAnalysis = options["onlyanalysis"] except KeyError: onlyAnalysis = False CCronLogger.logFile = logFile CCronLogger.logLevel = CCronLogger.setLogLevel(logLevel) CCronLogger.removeLog = removeLog CCronLogger.initLog() #parsovani crontabu cronParser = CCronParser(options) cronList = cronParser.parseCrontabs() crontabs = cronParser.getCrontabs() predictionDuration = checkPredictionDuration(options) timeInterval = timedelta(days=predictionDuration) now = datetime.datetime.now() then = now + timeInterval #predikce uloh i = 0 for cron in cronList: try: cron.predictUntil(then) except ValueError: errMsg = "Error occured while predicting following cronjob " + str( cron) + ", maybe job is badly configured" CCronLogger.errorToConsole(errMsg) CCronLogger.error(errMsg) #cron.printCron() i += 1 #zkontroluje prazdne cron joby cronList, emptyCronList = checkEmptyCronJobs(cronList) #vykreslovani uloh plotter = CCronPlotter(cronList, options) anyCrons, std, maxW, minW = plotter.plotCronJobs() if not anyCrons: exit(1) statsBefore = [std, maxW, minW] start = plotter.startTime end = plotter.endTime duration = plotter.duration if onlyAnalysis: exit(0) #zavolani optimalizatoru shiftVector, newCronList = optimize.main(cronList, options) CCronLogger.info("Starting to plot improved cron jobs.") CCronLogger.infoToConsole("Starting to plot improved cron jobs.") #vykresleni uloh po optimalizaci plotterAfter = CCronPlotter(newCronList, options, True) anyCrons, std, maxW, minW = plotterAfter.plotCronJobs() statsAfter = [std, maxW, minW] #vytvoreni noveho crontabu crontabCreator = CCrontabCreator(shiftVector, cronList, emptyCronList, crontabs) newCrontab = crontabCreator.createCrontab() reporter = CCronReport(options, newCrontab, statsBefore, statsAfter, shiftVector, cronList) reporter.createReport() CCronLogger.finishLog(statsBefore, statsAfter)
def setOptions(self, options): try: self.defaultLength = int(options["defaultduration"]) if self.defaultLength < 1: CCronLogger.error( "Invalid default duration. Setting to default: 1.") self.defaultLength = 1 CCronLogger.debug("Default weight: " + str(self.defaultLength) + ".") except ValueError: CCronLogger.error( "Invalid default duration. Setting to default: 1.") self.defaultLength = 1 except KeyError: CCronLogger.info( "Default duration directive not found. Setting to default: 1.") self.defaultLength = 1 try: self.defaultWeight = int(options["defaultweight"]) if self.defaultWeight < 1: CCronLogger.error( "Invalid default weight. Setting to default: 5.") self.defaultWeight = 5 CCronLogger.debug("Default weight: " + str(self.defaultWeight) + ".") except ValueError: CCronLogger.error("Invalid default weight. Setting to default: 5.") self.defaultWeight = 5 except KeyError: CCronLogger.info( "Default weight directive not found. Setting to default: 5.") self.defaultWeight = 5 try: self.defaultMaxShift = int(options["defaultmaxshift"]) CCronLogger.debug("Default MinShift: " + str(self.defaultMaxShift) + ".") except ValueError: CCronLogger.error( "Invalid default MaxShift. Setting to default: 0.") self.defaultMaxShift = 0 except KeyError: CCronLogger.info( "Default MaxShift directive not found. Setting to default: 0.") self.defaultMaxShift = 0 try: self.defaultMinShift = int(options["defaultminshift"]) CCronLogger.debug("Default MinShift: " + str(self.defaultMinShift) + ".") except ValueError: CCronLogger.error( "Invalid default MinShift. Setting to default: 0.") self.defaultMinShift = 0 except KeyError: CCronLogger.info( "Default MinShift directive not found. Setting to default: 0.") self.defaultMinShift = 0 if self.defaultMinShift > self.defaultMaxShift: CCronLogger.error( "Invalid default MinShift and MaxShift. Setting both 0.") self.defaultMaxShift = 0 self.defaultMinShift = 0 try: self.userCrontabDirs = options["usercrontabdirs"] CCronLogger.debug("User crontab directories: " + str(self.userCrontabDirs)) except KeyError: CCronLogger.info("User crontab directories directive not found.") self.userCrontabDirs = "" try: self.otherCrontabDirs = options["othercrontabdirs"] CCronLogger.debug("Other crontab directories: " + str(self.otherCrontabDirs)) except KeyError: CCronLogger.info("Other crontab directories directive not found.") self.otherCrontabDirs = "" try: self.includeRareJobs = options["includerarecronjobs"] CCronLogger.debug("Include rare cron jobs: " + str(self.includeRareJobs)) except KeyError: CCronLogger.info( "Include rare cron jobs directive not found. Setting to default: True." ) self.includeRareJobs = True try: self.includeMainCrontab = options["includemaincrontab"] CCronLogger.debug("Include main crontab: " + str(self.includeMainCrontab)) except KeyError: CCronLogger.info( "Include main crontab directive not found. Setting to default: False." ) self.includeMainCrontab = False try: self.includeCrondFolder = options["includecrondfolder"] CCronLogger.debug("Include /etc/cron.d/ folder: " + str(self.includeCrondFolder)) except KeyError: CCronLogger.info( "Include /etc/cron.d/ folder directive not found. Setting to default: False." ) self.includeCrondFolder = False
def _parseCrontab(self, content, sourceCrontab, isUser): CCronLogger.infoToConsole("Parsing crontabs...") CCronLogger.info("Parsing crontabs...") lineNr = 0 for line in content: #odstraneni bilych znaku z konce radky line = line.rstrip() #pokud je to radek zacinajici # = komentar nebo prazdna radka, tak ji preskoc if re.match("^[ \t]*#", line) or line == "": continue #REGEX na radku s cron ulohou #cronEntry=re.match("^[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^#\n]*)[ \t]*#?[ \t]*([^\n]*)$", line) cronEntry = line.split() #neni to crontab zaznam, pravdepodovne nastaveni promenne prostredi if (isUser and len(cronEntry) < 6) or (not isUser and len(cronEntry) < 7): continue #minute=cronEntry.group(1) #hour=cronEntry.group(2) #day=cronEntry.group(3) #month=cronEntry.group(4) #dayOfWeek=cronEntry.group(5) #command=cronEntry.group(6) #comment=cronEntry.group(7) minute = cronEntry[0] hour = cronEntry[1] day = cronEntry[2] month = cronEntry[3] dayOfWeek = cronEntry[4] if not self._checkMinute(minute) or not self._checkHour( hour) or not self._checkDay(day) or not self._checkMonth( month) or not self._checkDayOfWeek(dayOfWeek): msg = "Error while parsing crontab entry in file: " + sourceCrontab CCronLogger.errorToConsole(msg) CCronLogger.error(msg) continue if isUser: commandAndComment = cronEntry[5:] user = "" else: user = cronEntry[5] commandAndComment = cronEntry[6:] comment = [] command = [] commentStarted = False jednoducheUv = 0 dvojiteUv = 0 for word in commandAndComment: jednoducheUv += word.count("'") dvojiteUv += word.count('"') if re.search( "#", word) and not dvojiteUv % 2 and not jednoducheUv % 2: commentStarted = True if commentStarted: word = word.replace("#", "") words = word.split(",") for word in words: if word != "": comment.append(word) else: command.append(word) #check na neukoncene uvozovky if jednoducheUv % 2 or dvojiteUv % 2: msg = "Unfinished quotation marks, skipping cron entry" CCronLogger.errorToConsole(msg) CCronLogger.error(msg) continue command = " ".join(command) length = self._getMetadata(comment, "duration", lineNr, sourceCrontab) weight = self._getMetadata(comment, "weight", lineNr, sourceCrontab) shift = self._getMetadata(comment, "shift", lineNr, sourceCrontab) if length is None: length = self.defaultLength if weight is None: weight = self.defaultWeight if shift is None: shift = str(self.defaultMinShift) + ":" + str( self.defaultMaxShift) if shift is not None: shifts = shift.split(":") if len(shifts) < 2: leftShift = "-" + shift rightShift = shift else: leftShift = shifts[0] rightShift = shifts[1] leftShift, rightShift = self.checkShifts(leftShift, rightShift, line) weight = self.checkWeight(weight, line) length = self.checkLength(length, line) try: cron = CCron(minute, hour, day, month, dayOfWeek, user, command, int(length), int(weight), int(leftShift), int(rightShift), sourceCrontab, self.includeRareJobs) self._crons.append(cron) except ValueError: errMsg = "Error occured while predicting following cronjob: \n" errMsg = errMsg + minute + " " + hour + " " + day + " " + month + " " + dayOfWeek + " " + user + " " + command + "\n" errMsg = errMsg + "from crontab: " + sourceCrontab + "\n" errMsg = errMsg + "maybe job is badly configured or check \"include rare cron job\" directive" CCronLogger.errorToConsole(errMsg) CCronLogger.error(errMsg) lineNr += 1
def setOptions(self,options): try: self._threshold=int(options["threshold"]) if self._threshold < 0: CCronLogger.error("Invalid threshold. Setting to default: 30.") self._threshold=30 CCronLogger.debug("Threshold: "+str(self._threshold)) except ValueError: CCronLogger.error("Invalid threshold. Setting to default: 30.") self._threshold=30 except KeyError: CCronLogger.info("Threshold directive not found. Setting to default: 30.") self._threshold=30 try: self._duration=int(options["predictionduration"]) if self._duration < 1: CCronLogger.error("Invalid duration of prediction. Setting to default: 30.") self._duration=30 CCronLogger.debug("Duration of prediction: "+str(self._duration)+" days.") except ValueError: CCronLogger.error("Invalid duration of prediction. Setting to default: 30.") self._duration=30 except KeyError: CCronLogger.info("Duration of prediction directive not found. Setting to default: 30.") self._duration=30 try: self._intervalLen=int(options["testintervallength"]) if self._intervalLen < 1: CCronLogger.error("Invalid interval length. Setting to default: 1.") self._intervalLen=1 CCronLogger.debug("Interval length: "+str(self._intervalLen)) except ValueError: CCronLogger.error("Invalid interval length. Setting to default: 1.") self._intervalLen=1 except KeyError: CCronLogger.info("Interval length directive not found. Setting to default: 1.") self._intervalLen=1 try: self._samplesize=int(options["loadgraphsamplesize"]) if self._samplesize < 1: CCronLogger.error("Invalid sample size. Setting to default: 200.") self._samplesize=200 CCronLogger.debug("Sample size: "+str(self._samplesize)) except ValueError: CCronLogger.error("Invalid sample size. Setting to default: 200.") self._samplesize=200 except KeyError: CCronLogger.info("Sample size directive not found. Setting to default: 200.") self._samplesize=200 try: self._cronLenLimit=int(options["ignoreshortcronjobslimit"]) if self._cronLenLimit < 1: CCronLogger.error("Invalid short cron job limit. Setting to default: 30s.") self._cronLenLimit=30 CCronLogger.debug("Short cron job limit: "+str(self._cronLenLimit)+" seconds.") except ValueError: CCronLogger.error("Invalid short cron job limit. Setting to default: 30s.") self._cronLenLimit=30 except KeyError: CCronLogger.info("Short cron job limit directive not found. Setting to default: 30s.") self._cronLenLimit=30 try: self._showImages=options["showinteractiveimages"] CCronLogger.debug("Show interactive image: "+str(self._showImages)) except KeyError: CCronLogger.info("Show interactive images directive not found. Setting to default: True.") self._showImages=True try: self._ignoreShort=options["ignoreshortcronjobs"] CCronLogger.debug("Ignore short cron jobs: "+str(self._ignoreShort)) except KeyError: CCronLogger.info("Ignore short cron jobs directive not found. Setting to default: False.") self._ignoreShort=False try: self._outputDir=options["outputdir"] CCronLogger.debug("Output directory: "+self._outputDir) except KeyError: CCronLogger.info("Output directory directive not found. Setting to default: working directory.") self._outputDir="./" try: self._timeFormat=options["timeformatonxaxis"] try: _=datetime.datetime.now().strftime(self._timeFormat) except ValueError: CCronLogger.error("Invalid time format. Setting to default: %d.%m.%Y %H:%M:%S") self._timeFormat="%d.%m.%Y %H:%M:%S" CCronLogger.debug("Time format on X axis: "+self._timeFormat) except KeyError: CCronLogger.info("Time format on X axis directive not found. Setting to default: %d.%m.%Y %H:%M:%S") self._timeFormat="%d.%m.%Y %H:%M:%S" try: self.weightGraphLineWidth=float(options["weightgraphlinewidth"]) if self.weightGraphLineWidth < 0.5: CCronLogger.error("Invalid weight graph line width. Setting to default: 0.75.") self.weightGraphLineWidth=0.75 CCronLogger.debug("Weight graph line width: "+str(self.weightGraphLineWidth)) except ValueError: CCronLogger.error("Invalid weight graph line width. Setting to default: 0.75.") self.weightGraphLineWidth=0.75 except KeyError: CCronLogger.info("Weight graph line width directive not found. Setting to default: 0.75.") self.weightGraphLineWidth=0.75 try: self._graphName=options["graphname"] CCronLogger.debug("Graph name: "+self._graphName) except KeyError: CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.") self._graphName="graph.pdf" try: self._weightGraphName=options["weightgraphname"] CCronLogger.debug("Weight graph name: "+self._weightGraphName) except KeyError: CCronLogger.info("Weight graph name directive not found. Setting to default: weightGraph.pdf.") self._weightGraphName="weightGraph.pdf" try: self._approximation=options["loadgraphapproximation"] CCronLogger.debug("Weight graph approximaton: "+str(self._approximation)) except KeyError: CCronLogger.info("Weight graph directive not found. Setting to default: True.") self._approximation=True
def plotCronJobs(self): if len(self._cronList) == 0: CCronLogger.errorToConsole("No viable cron jobs to plot and analyze.") return False,None,None,None height_graph=2+ (len(self._cronList) / 2) #inicializace fig = plt.figure(figsize=(8,height_graph)) ax = fig.add_subplot(111) self._countTotalOverlaps() self._setLineWidth() firstTime=datetime.datetime.now().replace(year=9999) lastTime=datetime.datetime.now().replace(year=1) cronNameList=[] yLabels=[] cronCounter=0 cronID=1 counter=0 shown=False print("Plotting cron jobs.") for cron in self._cronList: if self._ignoreShort and cron.duration < timedelta(seconds=self._cronLenLimit): CCronLogger.debug("Cron job "+str(cron)+" is too short, it will be skipped.") continue #pokud je prikaz moc dlouhy, orizni ho if len(cron.command) > 30: cronNameList.append(cron.command[0:27]+" ...") else: cronNameList.append(cron.command) if cron.overlaps > 0: CCronLogger.info("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.") CCronLogger.infoToConsole("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.") yLabels.append(cronID + (cron.overlaps / 2) ) jobID=0 for job in cron.cronJobList: self._weights.append([job.startTime,job.endTime,cron.weight]) self._endPoints[job.startTime]=None self._endPoints[job.endTime]=None if job.startTime<firstTime: firstTime=job.startTime if job.endTime>lastTime: lastTime=job.endTime #cron joby se prekryvaji if cron.overlaps>0: #vykresli je nad sebe, cervene self._drawLine(cronID + (jobID % (cron.overlaps + 1) ),job.startTime,job.endTime,"r") #crony se neprekryvaji else: self._drawLine(cronID,job.startTime,job.endTime,"k") #prubeh progress=int(counter*100/self.cronJobsTotal) if not progress%5: if not shown: CCronLogger.debugToConsoleWithLineFeed("Progress: "+str(progress)+"%") shown=True else: shown=False counter+=1 jobID+=1 cronID+=(cron.overlaps+1) cronCounter+=1 #pokud to neni posledni vykreslovany cron, udelej vodorovnou caru if cronCounter!=len(self._cronList): plt.axhline(cronID - 0.5,color="black") CCronLogger.debugToConsole("Progress: 100%") if counter == 0: CCronLogger.error("No crons to plot.") CCronLogger.errorToConsole("No crons to plot.") return False,None,None,None #osaX = cas ax.xaxis_date() #format casu na ose length=lastTime-firstTime self.startTime=firstTime self.endTime=lastTime self.duration=length dateFormat = DateFormatter(self._timeFormat) #nastaveni osyX format casu ax.xaxis.set_major_formatter(dateFormat) #rezerva na oseX delta=(lastTime-firstTime)/100 #rozsah osy y plt.ylim(-0.5,cronID) #rozsah osyX (zacatecni,konecny cas) + nejaka rezerva plt.xlim(firstTime-delta, lastTime+delta) #popis osyX plt.xlabel('Time') #plt.ylabel("Cron jobs") fig.autofmt_xdate() centers, weights, maxWeight, minWeight, summedWeights,weightsForStatistics = self.makeWeightIntervals() std=self.createStatistics(maxWeight, weightsForStatistics) loadPeaks = self.findLoadPeaks(summedWeights) if len(loadPeaks) > 0: yLabels.insert(0, 0) cronNameList.insert(0, "Bottlenecks") plt.axhline(0.5,color="black",lw=2.5) loadPeaksLen=timedelta(seconds=0) totalWeight=0 for peak in loadPeaks: plt.hlines(0, peak[0], peak[1], "r", lw=self.lineWidth) startTime=peak[0].strftime("%d.%m.%Y %H:%M:%S") endTime=peak[1].strftime("%d.%m.%Y %H:%M:%S") totalWeight+= peak[2] loadPeaksLen+=(peak[1]-peak[0]) CCronLogger.info("Found bottleneck: "+startTime+" <---> "+endTime) CCronLogger.info("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight)) CCronLogger.infoToConsole("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight)) #popisky osyY: 1.arg list ID (unikatni cronjoby), 2.arg = list popisku (nazvy cronjobu) plt.yticks(yLabels, cronNameList) #autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno plt.tight_layout() #export do pdf if self.plotBetter: CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName) CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName) try: savefig(self._outputDir+"improved_"+self._graphName) except: CCronLogger.errorToConsole("Error while saving image.") CCronLogger.error("Error while saving image.") else: CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+self._graphName) CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+self._graphName) try: savefig(self._outputDir+self._graphName) except: CCronLogger.errorToConsole("Error while saving image.") CCronLogger.error("Error while saving image.") #ukazani grafu v Tk if self._showImages: plt.show() self.plotWeightGraph(length, centers, weights, maxWeight) return True,std,maxWeight,minWeight
def plotWeightGraph(self, length,centers, weights, maxWeight): #inicializace width = int(self._duration*10) if width > 25: width=25 fig = plt.figure(figsize=(width ,6)) ax = fig.add_subplot(111) thresholds=asarray([self._threshold]*len(weights)) plt.plot(centers,weights,color="k",lw=self.weightGraphLineWidth) plt.fill_between(centers,weights,thresholds,where=weights>thresholds,interpolate=True,color="r") ax.xaxis_date() dateFormat = DateFormatter(self._timeFormat) #nastaveni osyX format casu ax.xaxis.set_major_formatter(dateFormat) #rezerva na oseX #rozsah osyX (zacatecni,konecny cas) + nejaka rezerva #popis osyX plt.xlabel('Time') plt.ylabel("Load") fig.autofmt_xdate() padd=maxWeight/10 plt.ylim(0,maxWeight+padd) #autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno plt.tight_layout() #export do pdf if self.plotBetter: CCronLogger.info("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName) CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName) try: savefig(self._outputDir+"improved_"+self._weightGraphName) except: CCronLogger.errorToConsole("Error while saving image.") CCronLogger.error("Error while saving image.") else: CCronLogger.info("Saving plot with weights to: "+self._outputDir+self._weightGraphName) CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+self._weightGraphName) try: savefig(self._outputDir+self._weightGraphName) except: CCronLogger.errorToConsole("Error while saving image.") CCronLogger.error("Error while saving image.") #ukazani grafu v Tk if self._showImages: plt.show()
def setOptions(self, options): try: self._intervals = int(options["intervalsperday"]) CCronLogger.debug("Intervals per day: " + str(self._intervals)) except ValueError: CCronLogger.error( "Invalid intervals per day. Setting to default: 10.") self._intervals = 10 except KeyError: CCronLogger.info( "Intervals per day directive not found. Setting to default: 10." ) self._intervals = 10 try: self._duration = int(options["predictionduration"]) CCronLogger.debug("Duration of prediction: " + str(self._duration) + " days.") except ValueError: CCronLogger.error( "Invalid duration of prediction. Setting to default: 30.") self._duration = 30 except KeyError: CCronLogger.info( "Duration of prediction directive not found. Setting to default: 30." ) self._duration = 30 try: self._numberOfGenerations = int(options["numberofgenerations"]) CCronLogger.debug("Number of generations: " + str(self._numberOfGenerations) + " generations.") except ValueError: CCronLogger.error( "Invalid number of generations. Setting to default: 100.") self._numberOfGenerations = 100 except KeyError: CCronLogger.info( "Number of generations directive not found. Setting to default: 100." ) self._numberOfGenerations = 100 try: self._sizeOfPopulation = int(options["sizeofpopulation"]) CCronLogger.debug("Size of population: " + str(self._sizeOfPopulation) + ".") except ValueError: CCronLogger.error( "Invalid size of populaton. Setting to default: 75.") self._sizeOfPopulation = 75 except KeyError: CCronLogger.info( "Size of population directive not found. Setting to default: 75." ) self._sizeOfPopulation = 75 try: self._crossoverProb = float(options["crossoverprobability"]) CCronLogger.debug("Crossover probability: " + str(self._crossoverProb * 100) + " %.") except ValueError: CCronLogger.error( "Invalid crossover probability. Setting to default: 0.75.") self._crossoverProb = 0.75 except KeyError: CCronLogger.info( "Invalid probability directive not found. Setting to default: 0.75." ) self._crossoverProb = 0.75 try: self._mutationProb = float(options["mutationprobability"]) CCronLogger.debug("Mutation probability: " + str(self._mutationProb * 100) + " %.") except ValueError: CCronLogger.error( "Invalid mutation probability. Setting to default: 0.05.") self._mutationProb = 0.05 except KeyError: CCronLogger.info( "Mutation probability directive not found. Setting to default: 0.05." ) self._mutationProb = 0.05
def setOptions(options): try: intervalLen = int(options["testintervallength"]) if intervalLen < 1: CCronLogger.error( "Invalid interval length. Setting to default: 1.") intervalLen = 1 CCronLogger.debug("Interval length: " + str(intervalLen)) except ValueError: CCronLogger.error("Invalid interval length. Setting to default: 1.") intervalLen = 1 except KeyError: CCronLogger.info( "Interval length directive not found. Setting to default: 1.") intervalLen = 1 try: mateOp = options["mateoperator"] mateOp = mateOp.lower() if mateOp == "onepoint" or mateOp == "twopoint" or mateOp == "uniform": CCronLogger.debug("Mate operator: " + str(mateOp)) else: CCronLogger.error( "Unknown value for mate operator. Setting to default: OnePoint." ) mateOp = "onepoint" except KeyError: CCronLogger.info( "Mate operator directive not found. Setting to default: OnePoint.") mateOp = "onepoint" try: parallel = options["parallelcomputation"] CCronLogger.debug("Parallel computation: " + str(parallel)) except KeyError: CCronLogger.info( "Parallel computation directive not found. Setting to default: false." ) parallel = False try: duration = int(options["predictionduration"]) if duration < 1: CCronLogger.error( "Invalid duration of prediction. Setting to default: 30.") duration = 30 CCronLogger.debug("Duration of prediction: " + str(duration) + " days.") except ValueError: CCronLogger.error( "Invalid duration of prediction. Setting to default: 30.") duration = 30 except KeyError: CCronLogger.info( "Duration of prediction directive not found. Setting to default: 30." ) duration = 30 try: numberOfGenerations = int(options["numberofgenerations"]) if numberOfGenerations < 1: CCronLogger.error( "Invalid number of generations. Setting to default: 100.") numberOfGenerations = 100 CCronLogger.debug("Number of generations: " + str(numberOfGenerations) + " generations.") except ValueError: CCronLogger.error( "Invalid number of generations. Setting to default: 100.") numberOfGenerations = 100 except KeyError: CCronLogger.info( "Number of generations directive not found. Setting to default: 100." ) numberOfGenerations = 100 try: sizeOfPopulation = int(options["sizeofpopulation"]) if sizeOfPopulation < 1: CCronLogger.error( "Invalid size of populaton. Setting to default: 75.") sizeOfPopulation = 75 CCronLogger.debug("Size of population: " + str(sizeOfPopulation) + ".") except ValueError: CCronLogger.error("Invalid size of populaton. Setting to default: 75.") sizeOfPopulation = 75 except KeyError: CCronLogger.info( "Size of population directive not found. Setting to default: 75.") sizeOfPopulation = 75 try: crossoverProb = float(options["crossoverprobability"]) if crossoverProb < 0: CCronLogger.error( "Invalid crossover probability. Setting to default: 0.75.") crossoverProb = 0.75 CCronLogger.debug("Crossover probability: " + str(crossoverProb * 100) + " %.") except ValueError: CCronLogger.error( "Invalid crossover probability. Setting to default: 0.75.") crossoverProb = 0.75 except KeyError: CCronLogger.info( "Invalid probability directive not found. Setting to default: 0.75." ) crossoverProb = 0.75 try: mutationProb = float(options["mutationprobability"]) if mutationProb < 0: CCronLogger.error( "Invalid mutation probability. Setting to default: 0.05.") mutationProb = 0.05 CCronLogger.debug("Mutation probability: " + str(mutationProb * 100) + " %.") except ValueError: CCronLogger.error( "Invalid mutation probability. Setting to default: 0.05.") mutationProb = 0.05 except KeyError: CCronLogger.info( "Mutation probability directive not found. Setting to default: 0.05." ) mutationProb = 0.05 try: uniformProb = float(options["uniformoperatorprobability"]) if uniformProb < 0: CCronLogger.error( "Invalid Uniform crossover probability. Setting to default: 0.75." ) uniformProb = 0.75 CCronLogger.debug("Uniform crossover probability: " + str(uniformProb * 100) + " %.") except ValueError: CCronLogger.error( "Invalid Uniform crossover probability. Setting to default: 0.75.") uniformProb = 0.75 except KeyError: CCronLogger.info( "Uniform crossover probability directive not found. Setting to default: 0.75." ) uniformProb = 0.75 try: tournamentSize = int(options["tournamentsize"]) if tournamentSize < 2: CCronLogger.error( "Invalid tournament size. Setting to default: 3.") tournamentSize = 3 CCronLogger.debug("Tournament size: " + str(tournamentSize) + ".") except ValueError: CCronLogger.error("Invalid tournament size. Setting to default: 3.") tournamentSize = 3 except KeyError: CCronLogger.info( "Tournament size directive not found. Setting to default: 3.") tournamentSize = 3 return intervalLen, duration, numberOfGenerations, sizeOfPopulation, crossoverProb, mutationProb, parallel, mateOp, uniformProb, tournamentSize
def optimize(cronList, minShifts, maxShifts, sizeOfPopulation, numberOfGenerations, crossoverProb, mutationProb, parallel, mateOp, uniformProb, tournamentSize): #zaregistrovani fitness creator.create("FitnessMinLoad", base.Fitness, weights=(-1, )) #vytvoreni jedince creator.create("Individual", list, fitness=creator.FitnessMinLoad) toolbox = base.Toolbox() toolbox.register("gene", genRandomShift) # vytvoreni listu jedincu toolbox.register("individual", customInitRepeat, creator.Individual, toolbox.gene, len(cronList)) toolbox.register("population", tools.initRepeat, list, toolbox.individual) # nastaveni fitness fce toolbox.register("evaluate", evalOneMinLoad) #nastaveni mate operatoru if mateOp == "uniform": toolbox.register("mate", tools.cxUniform, indpb=uniformProb) elif mateOp == "onepoint": toolbox.register("mate", tools.cxOnePoint) elif mateOp == "twopoint": toolbox.register("mate", tools.cxTwoPoint) #nastaveni operatoru mutace toolbox.register("mutate", tools.mutUniformInt, low=minShifts, up=maxShifts, indpb=1 / len(cronList)) toolbox.register("select", tools.selTournament, tournsize=tournamentSize) pool = multiprocessing.Pool() #vymena map za paralelni map if parallel: toolbox.register("map", pool.map) else: toolbox.register("map", map) #random.seed(64) random.seed(os.urandom(100)) pop = toolbox.population(n=sizeOfPopulation) CXPB, MUTPB, NGEN = crossoverProb, mutationProb, numberOfGenerations CCronLogger.infoToConsole("Start of evolution") CCronLogger.info("Start of evolution") # ohodnoceni cele populace fitnesses = list(toolbox.map(evalOneMinLoad, pop)) for ind, fit in zip(pop, fitnesses): ind.fitness.values = fit CCronLogger.infoToConsole(" Evaluated %i individuals" % len(pop)) CCronLogger.info(" Evaluated %i individuals" % len(pop)) # zacatek evoluce for g in range(NGEN): CCronLogger.infoToConsole("-- Generation %i --" % g) CCronLogger.info("-- Generation %i --" % g) # vyber jedincu do dalsi populace offspring = toolbox.select(pop, len(pop)) # klonovani vybranych jedincu offspring = list(toolbox.map(toolbox.clone, offspring)) # aplikovani operatoru mutace a krizeni for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # ohodnoceni jedincu invalid_ind = [ind for ind in offspring if not ind.fitness.valid] fitnesses = toolbox.map(evalOneMinLoad, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit CCronLogger.infoToConsole(" Evaluated %i individuals" % len(invalid_ind)) CCronLogger.info(" Evaluated %i individuals" % len(invalid_ind)) pop[:] = offspring fits = [ind.fitness.values[0] for ind in pop] length = len(pop) mean = sum(fits) / length sum2 = sum(x * x for x in fits) std = abs(sum2 / length - mean**2)**0.5 CCronLogger.infoToConsole(" Min %s" % min(fits)) CCronLogger.info(" Min %s" % min(fits)) CCronLogger.infoToConsole(" Max %s" % max(fits)) CCronLogger.info(" Max %s" % max(fits)) CCronLogger.infoToConsole(" Avg %s" % mean) CCronLogger.info(" Avg %s" % mean) CCronLogger.infoToConsole(" Std %s" % std) CCronLogger.info(" Std %s" % std) myMax = -1 if max(fits) > myMax: myMax = max(fits) CCronLogger.infoToConsole("-- End of (successful) evolution --") CCronLogger.info("-- End of (successful) evolution --") best_ind = tools.selBest(pop, 1)[0] CCronLogger.infoToConsole("Best individual is %s, %s" % (best_ind, best_ind.fitness.values)) CCronLogger.info("Best individual is %s, %s" % (best_ind, best_ind.fitness.values)) cronList = applyBestShift(best_ind, cronList) return best_ind, cronList