def chooseRandomSample(self,centers,weights):
centersAndWeights=[]
#spojim do jednoho pole
for i in range(0,len(centers)):
centersAndWeights.append([centers[i],weights[i]])
#vyberu vzorek
if self._approximation:
try:
sample=random.sample(centersAndWeights,self._samplesize)
except ValueError:
CCronLogger.info("Sample larger than population. Using all data.")
CCronLogger.infoToConsole("Sample larger than population. Using all data.")
sample=centersAndWeights
sample=sorted(sample,key=getKey)
else:
sample=centersAndWeights
#roztrhnu do dvou poli
centers=[]
weights=[]
for center, weight in sample:
centers.append(center)
weights.append(weight)
return centers, weights
def createStatistics(self,maxWeight,weights):
std=round(statistics.pstdev(weights),1)
msg="Standard deviation of weights in test intervals: "+str(std)
CCronLogger.infoToConsole(msg)
CCronLogger.info(msg)
msg="Max weight in test interval: "+str(maxWeight)
CCronLogger.infoToConsole(msg)
CCronLogger.info(msg)
return std
def __init__(self, options):
self.setOptions(options)
self._crontabFolders = []
#pridat slozky s crontaby do seznamu
#obecna slozka s crontaby
if self.includeCrondFolder:
self._crontabFolders.append(["/etc/cron.d/", False])
#uzivatelske slozky s crontaby
if os.geteuid() == 0:
for usercrontabdir in self.userCrontabDirs.split(":"):
self._crontabFolders.append([usercrontabdir, True])
else:
CCronLogger.info(
"You have to run this tool as root to access user crontabs.")
CCronLogger.infoToConsole(
"You have to run this tool as root to access user crontabs.")
#ostatni slozky s crontaby, ktere nejsou uzivatelske
for othercrontabdir in self.otherCrontabDirs.split(":"):
if othercrontabdir != "":
self._crontabFolders.append([othercrontabdir, False])
#muj testovaci crontab
#self._crontabFolders.append(["/home/petr/git/cronalyzer/data/crontabs/",True])
self._crontabs = []
for crontabFolder, isUser in self._crontabFolders:
for crontab in glob.glob(os.path.join(crontabFolder, "*")):
if not crontab.endswith("~") and os.path.isfile(crontab):
self._crontabs.append([crontab, isUser])
CCronLogger.info("Added crontab: " + crontab +
" to analysis.")
CCronLogger.infoToConsole("Added crontab: " + crontab +
" to analysis.")
# + /etc/crontab
if self.includeMainCrontab:
self._crontabs.append(["/etc/crontab", False])
CCronLogger.info("Added crontab: /etc/crontab to analysis.")
CCronLogger.infoToConsole(
"Added crontab: /etc/crontab to analysis.")
#list s rozparsovanymi crony
self._crons = []
def main():
#pokud neni zadan skript, zkusi se doplnit z aktualniho adresare
if len(sys.argv[1:]) == 0:
configFile = "config.conf"
else:
configFile = readArguments(sys.argv[1:])
testConfigFile(configFile)
configParser = CConfigParser(configFile)
options = configParser.parseConfig()
#nacteni a vytvoreni slozky
try:
outputDir = options["outputdir"]
except KeyError:
outputDir = "."
options["outputdir"] = repairOutputDir(outputDir)
createOutputDir(outputDir)
logFile = getLogFile(options)
checkLogFile(logFile)
try:
logLevel = options["loglevel"]
except KeyError:
logFile = "info"
try:
removeLog = options["removelogonstart"]
except KeyError:
removeLog = False
try:
onlyAnalysis = options["onlyanalysis"]
except KeyError:
onlyAnalysis = False
CCronLogger.logFile = logFile
CCronLogger.logLevel = CCronLogger.setLogLevel(logLevel)
CCronLogger.removeLog = removeLog
CCronLogger.initLog()
#parsovani crontabu
cronParser = CCronParser(options)
cronList = cronParser.parseCrontabs()
crontabs = cronParser.getCrontabs()
predictionDuration = checkPredictionDuration(options)
timeInterval = timedelta(days=predictionDuration)
now = datetime.datetime.now()
then = now + timeInterval
#predikce uloh
i = 0
for cron in cronList:
try:
cron.predictUntil(then)
except ValueError:
errMsg = "Error occured while predicting following cronjob " + str(
cron) + ", maybe job is badly configured"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
#cron.printCron()
i += 1
#zkontroluje prazdne cron joby
cronList, emptyCronList = checkEmptyCronJobs(cronList)
#vykreslovani uloh
plotter = CCronPlotter(cronList, options)
anyCrons, std, maxW, minW = plotter.plotCronJobs()
if not anyCrons:
exit(1)
statsBefore = [std, maxW, minW]
start = plotter.startTime
end = plotter.endTime
duration = plotter.duration
if onlyAnalysis:
exit(0)
#zavolani optimalizatoru
shiftVector, newCronList = optimize.main(cronList, options)
CCronLogger.info("Starting to plot improved cron jobs.")
CCronLogger.infoToConsole("Starting to plot improved cron jobs.")
#vykresleni uloh po optimalizaci
plotterAfter = CCronPlotter(newCronList, options, True)
anyCrons, std, maxW, minW = plotterAfter.plotCronJobs()
statsAfter = [std, maxW, minW]
#vytvoreni noveho crontabu
crontabCreator = CCrontabCreator(shiftVector, cronList, emptyCronList,
crontabs)
newCrontab = crontabCreator.createCrontab()
reporter = CCronReport(options, newCrontab, statsBefore, statsAfter,
shiftVector, cronList)
reporter.createReport()
CCronLogger.finishLog(statsBefore, statsAfter)
def _parseCrontab(self, content, sourceCrontab, isUser):
CCronLogger.infoToConsole("Parsing crontabs...")
CCronLogger.info("Parsing crontabs...")
lineNr = 0
for line in content:
#odstraneni bilych znaku z konce radky
line = line.rstrip()
#pokud je to radek zacinajici # = komentar nebo prazdna radka, tak ji preskoc
if re.match("^[ \t]*#", line) or line == "":
continue
#REGEX na radku s cron ulohou
#cronEntry=re.match("^[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^#\n]*)[ \t]*#?[ \t]*([^\n]*)$", line)
cronEntry = line.split()
#neni to crontab zaznam, pravdepodovne nastaveni promenne prostredi
if (isUser and len(cronEntry) < 6) or (not isUser
and len(cronEntry) < 7):
continue
#minute=cronEntry.group(1)
#hour=cronEntry.group(2)
#day=cronEntry.group(3)
#month=cronEntry.group(4)
#dayOfWeek=cronEntry.group(5)
#command=cronEntry.group(6)
#comment=cronEntry.group(7)
minute = cronEntry[0]
hour = cronEntry[1]
day = cronEntry[2]
month = cronEntry[3]
dayOfWeek = cronEntry[4]
if not self._checkMinute(minute) or not self._checkHour(
hour) or not self._checkDay(day) or not self._checkMonth(
month) or not self._checkDayOfWeek(dayOfWeek):
msg = "Error while parsing crontab entry in file: " + sourceCrontab
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
if isUser:
commandAndComment = cronEntry[5:]
user = ""
else:
user = cronEntry[5]
commandAndComment = cronEntry[6:]
comment = []
command = []
commentStarted = False
jednoducheUv = 0
dvojiteUv = 0
for word in commandAndComment:
jednoducheUv += word.count("'")
dvojiteUv += word.count('"')
if re.search(
"#",
word) and not dvojiteUv % 2 and not jednoducheUv % 2:
commentStarted = True
if commentStarted:
word = word.replace("#", "")
words = word.split(",")
for word in words:
if word != "":
comment.append(word)
else:
command.append(word)
#check na neukoncene uvozovky
if jednoducheUv % 2 or dvojiteUv % 2:
msg = "Unfinished quotation marks, skipping cron entry"
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
command = " ".join(command)
length = self._getMetadata(comment, "duration", lineNr,
sourceCrontab)
weight = self._getMetadata(comment, "weight", lineNr,
sourceCrontab)
shift = self._getMetadata(comment, "shift", lineNr, sourceCrontab)
if length is None:
length = self.defaultLength
if weight is None:
weight = self.defaultWeight
if shift is None:
shift = str(self.defaultMinShift) + ":" + str(
self.defaultMaxShift)
if shift is not None:
shifts = shift.split(":")
if len(shifts) < 2:
leftShift = "-" + shift
rightShift = shift
else:
leftShift = shifts[0]
rightShift = shifts[1]
leftShift, rightShift = self.checkShifts(leftShift, rightShift,
line)
weight = self.checkWeight(weight, line)
length = self.checkLength(length, line)
try:
cron = CCron(minute, hour, day, month, dayOfWeek, user,
command, int(length), int(weight), int(leftShift),
int(rightShift), sourceCrontab,
self.includeRareJobs)
self._crons.append(cron)
except ValueError:
errMsg = "Error occured while predicting following cronjob: \n"
errMsg = errMsg + minute + " " + hour + " " + day + " " + month + " " + dayOfWeek + " " + user + " " + command + "\n"
errMsg = errMsg + "from crontab: " + sourceCrontab + "\n"
errMsg = errMsg + "maybe job is badly configured or check \"include rare cron job\" directive"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
lineNr += 1
def plotCronJobs(self):
if len(self._cronList) == 0:
CCronLogger.errorToConsole("No viable cron jobs to plot and analyze.")
return False,None,None,None
height_graph=2+ (len(self._cronList) / 2)
#inicializace
fig = plt.figure(figsize=(8,height_graph))
ax = fig.add_subplot(111)
self._countTotalOverlaps()
self._setLineWidth()
firstTime=datetime.datetime.now().replace(year=9999)
lastTime=datetime.datetime.now().replace(year=1)
cronNameList=[]
yLabels=[]
cronCounter=0
cronID=1
counter=0
shown=False
print("Plotting cron jobs.")
for cron in self._cronList:
if self._ignoreShort and cron.duration < timedelta(seconds=self._cronLenLimit):
CCronLogger.debug("Cron job "+str(cron)+" is too short, it will be skipped.")
continue
#pokud je prikaz moc dlouhy, orizni ho
if len(cron.command) > 30:
cronNameList.append(cron.command[0:27]+" ...")
else:
cronNameList.append(cron.command)
if cron.overlaps > 0:
CCronLogger.info("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
CCronLogger.infoToConsole("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
yLabels.append(cronID + (cron.overlaps / 2) )
jobID=0
for job in cron.cronJobList:
self._weights.append([job.startTime,job.endTime,cron.weight])
self._endPoints[job.startTime]=None
self._endPoints[job.endTime]=None
if job.startTime<firstTime:
firstTime=job.startTime
if job.endTime>lastTime:
lastTime=job.endTime
#cron joby se prekryvaji
if cron.overlaps>0:
#vykresli je nad sebe, cervene
self._drawLine(cronID + (jobID % (cron.overlaps + 1) ),job.startTime,job.endTime,"r")
#crony se neprekryvaji
else:
self._drawLine(cronID,job.startTime,job.endTime,"k")
#prubeh
progress=int(counter*100/self.cronJobsTotal)
if not progress%5:
if not shown:
CCronLogger.debugToConsoleWithLineFeed("Progress: "+str(progress)+"%")
shown=True
else:
shown=False
counter+=1
jobID+=1
cronID+=(cron.overlaps+1)
cronCounter+=1
#pokud to neni posledni vykreslovany cron, udelej vodorovnou caru
if cronCounter!=len(self._cronList):
plt.axhline(cronID - 0.5,color="black")
CCronLogger.debugToConsole("Progress: 100%")
if counter == 0:
CCronLogger.error("No crons to plot.")
CCronLogger.errorToConsole("No crons to plot.")
return False,None,None,None
#osaX = cas
ax.xaxis_date()
#format casu na ose
length=lastTime-firstTime
self.startTime=firstTime
self.endTime=lastTime
self.duration=length
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
delta=(lastTime-firstTime)/100
#rozsah osy y
plt.ylim(-0.5,cronID)
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
plt.xlim(firstTime-delta, lastTime+delta)
#popis osyX
plt.xlabel('Time')
#plt.ylabel("Cron jobs")
fig.autofmt_xdate()
centers, weights, maxWeight, minWeight, summedWeights,weightsForStatistics = self.makeWeightIntervals()
std=self.createStatistics(maxWeight, weightsForStatistics)
loadPeaks = self.findLoadPeaks(summedWeights)
if len(loadPeaks) > 0:
yLabels.insert(0, 0)
cronNameList.insert(0, "Bottlenecks")
plt.axhline(0.5,color="black",lw=2.5)
loadPeaksLen=timedelta(seconds=0)
totalWeight=0
for peak in loadPeaks:
plt.hlines(0, peak[0], peak[1], "r", lw=self.lineWidth)
startTime=peak[0].strftime("%d.%m.%Y %H:%M:%S")
endTime=peak[1].strftime("%d.%m.%Y %H:%M:%S")
totalWeight+= peak[2]
loadPeaksLen+=(peak[1]-peak[0])
CCronLogger.info("Found bottleneck: "+startTime+" <---> "+endTime)
CCronLogger.info("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
CCronLogger.infoToConsole("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
#popisky osyY: 1.arg list ID (unikatni cronjoby), 2.arg = list popisku (nazvy cronjobu)
plt.yticks(yLabels, cronNameList)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
try:
savefig(self._outputDir+"improved_"+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
try:
savefig(self._outputDir+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()
self.plotWeightGraph(length, centers, weights, maxWeight)
return True,std,maxWeight,minWeight
def plotWeightGraph(self, length,centers, weights, maxWeight):
#inicializace
width = int(self._duration*10)
if width > 25:
width=25
fig = plt.figure(figsize=(width ,6))
ax = fig.add_subplot(111)
thresholds=asarray([self._threshold]*len(weights))
plt.plot(centers,weights,color="k",lw=self.weightGraphLineWidth)
plt.fill_between(centers,weights,thresholds,where=weights>thresholds,interpolate=True,color="r")
ax.xaxis_date()
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
#popis osyX
plt.xlabel('Time')
plt.ylabel("Load")
fig.autofmt_xdate()
padd=maxWeight/10
plt.ylim(0,maxWeight+padd)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
try:
savefig(self._outputDir+"improved_"+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
try:
savefig(self._outputDir+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()
def handler(signum, frame):
CCronLogger.infoToConsole("Closing program.")
exit(0)
def optimize(cronList, minShifts, maxShifts, sizeOfPopulation,
numberOfGenerations, crossoverProb, mutationProb, parallel,
mateOp, uniformProb, tournamentSize):
#zaregistrovani fitness
creator.create("FitnessMinLoad", base.Fitness, weights=(-1, ))
#vytvoreni jedince
creator.create("Individual", list, fitness=creator.FitnessMinLoad)
toolbox = base.Toolbox()
toolbox.register("gene", genRandomShift)
# vytvoreni listu jedincu
toolbox.register("individual", customInitRepeat, creator.Individual,
toolbox.gene, len(cronList))
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
# nastaveni fitness fce
toolbox.register("evaluate", evalOneMinLoad)
#nastaveni mate operatoru
if mateOp == "uniform":
toolbox.register("mate", tools.cxUniform, indpb=uniformProb)
elif mateOp == "onepoint":
toolbox.register("mate", tools.cxOnePoint)
elif mateOp == "twopoint":
toolbox.register("mate", tools.cxTwoPoint)
#nastaveni operatoru mutace
toolbox.register("mutate",
tools.mutUniformInt,
low=minShifts,
up=maxShifts,
indpb=1 / len(cronList))
toolbox.register("select", tools.selTournament, tournsize=tournamentSize)
pool = multiprocessing.Pool()
#vymena map za paralelni map
if parallel:
toolbox.register("map", pool.map)
else:
toolbox.register("map", map)
#random.seed(64)
random.seed(os.urandom(100))
pop = toolbox.population(n=sizeOfPopulation)
CXPB, MUTPB, NGEN = crossoverProb, mutationProb, numberOfGenerations
CCronLogger.infoToConsole("Start of evolution")
CCronLogger.info("Start of evolution")
# ohodnoceni cele populace
fitnesses = list(toolbox.map(evalOneMinLoad, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
CCronLogger.infoToConsole(" Evaluated %i individuals" % len(pop))
CCronLogger.info(" Evaluated %i individuals" % len(pop))
# zacatek evoluce
for g in range(NGEN):
CCronLogger.infoToConsole("-- Generation %i --" % g)
CCronLogger.info("-- Generation %i --" % g)
# vyber jedincu do dalsi populace
offspring = toolbox.select(pop, len(pop))
# klonovani vybranych jedincu
offspring = list(toolbox.map(toolbox.clone, offspring))
# aplikovani operatoru mutace a krizeni
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# ohodnoceni jedincu
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(evalOneMinLoad, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
CCronLogger.infoToConsole(" Evaluated %i individuals" %
len(invalid_ind))
CCronLogger.info(" Evaluated %i individuals" % len(invalid_ind))
pop[:] = offspring
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x * x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
CCronLogger.infoToConsole(" Min %s" % min(fits))
CCronLogger.info(" Min %s" % min(fits))
CCronLogger.infoToConsole(" Max %s" % max(fits))
CCronLogger.info(" Max %s" % max(fits))
CCronLogger.infoToConsole(" Avg %s" % mean)
CCronLogger.info(" Avg %s" % mean)
CCronLogger.infoToConsole(" Std %s" % std)
CCronLogger.info(" Std %s" % std)
myMax = -1
if max(fits) > myMax:
myMax = max(fits)
CCronLogger.infoToConsole("-- End of (successful) evolution --")
CCronLogger.info("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
CCronLogger.infoToConsole("Best individual is %s, %s" %
(best_ind, best_ind.fitness.values))
CCronLogger.info("Best individual is %s, %s" %
(best_ind, best_ind.fitness.values))
cronList = applyBestShift(best_ind, cronList)
return best_ind, cronList