def _getMetadata(self, comment, metadata, lineNr, sourceCrontab):
data = None
for i in range(0, len(comment)):
meta = comment[i]
if meta.startswith(metadata + "="):
_, value = meta.split("=")
if value != "":
return value
try:
idx = comment.index(metadata)
if comment[idx + 1] != "=":
data = comment[idx + 1]
data = data.replace("=", "")
else:
data = comment[idx + 2]
except ValueError:
try:
idx = comment.index(metadata + "=")
data = comment[idx + 1]
except ValueError:
CCronLogger.info(metadata.title() +
" metadata not found in line " + str(lineNr) +
" in " + sourceCrontab + ".")
CCronLogger.info("It will be replaced by default value.")
return data
def prepareWeightList(cronList):
cronID = 0
origWeights = []
maxShifts = []
minShifts = []
for cron in cronList:
leftShift = cron.leftShift
rightShift = cron.rightShift
cronJobList = cron.cronJobList
weight = cron.weight
if len(cronJobList) == 0:
continue
maxShifts.append(rightShift)
minShifts.append(leftShift)
for cronJob in cronJobList:
start = cronJob.startTime
end = cronJob.endTime
origWeights.append([start, end, weight, cronID])
cronID += 1
CCronLogger.debugToConsole("Total number of cron jobs: " +
str(len(origWeights)))
origWeights = sorted(origWeights, key=getKey)
return [origWeights, minShifts, maxShifts]
def chooseRandomSample(self,centers,weights):
centersAndWeights=[]
#spojim do jednoho pole
for i in range(0,len(centers)):
centersAndWeights.append([centers[i],weights[i]])
#vyberu vzorek
if self._approximation:
try:
sample=random.sample(centersAndWeights,self._samplesize)
except ValueError:
CCronLogger.info("Sample larger than population. Using all data.")
CCronLogger.infoToConsole("Sample larger than population. Using all data.")
sample=centersAndWeights
sample=sorted(sample,key=getKey)
else:
sample=centersAndWeights
#roztrhnu do dvou poli
centers=[]
weights=[]
for center, weight in sample:
centers.append(center)
weights.append(weight)
return centers, weights
def createReport(self):
document = dominate.document(title="Ouput report from Cronalyzer")
document.head.add(meta(charset="utf-8"))
document.body.add(h1("Output report from Cronalyzer"))
document.body.add(h2("Plot with cronjobs before optimization:"))
document.body.add(img(src=self._graphName, alt="Plot with cronjobs."))
document.body.add(h2("Plot with weights of cronjobs before optimization:"))
document.body.add(img(src=self._weightGraphName, alt="Plot with weights of cronjobs."))
document.body.add(h2("Plot with cronjobs after optimization:"))
document.body.add(img(src="improved_"+self._graphName, alt="Plot with cronjobs after optimization."))
document.body.add(h2("Plot with weights of cronjobs after optimization:"))
document.body.add(img(src="improved_"+self._weightGraphName, alt="Plot with weights of cronjobs after optimization."))
document.body.add(h2("List of jobs and recommended shift in minutes"))
vector=document.body.add(div())
cronID=0
for cron in self.cronList:
shiftAmount=self.shiftVector[cronID]
vector += str(cron)+" should be shift by: "+str(shiftAmount)+" minutes."
vector += br()
cronID+=1
document.body.add(h2("Summary of optimization"))
document.body.add(h3("Statistics before optimization"))
before=document.body.add(div())
before += "Standard deviation in test intervals: "+str(self.statsBefore[0])
before += br()
before += "Max weight in test intervals: "+str(self.statsBefore[1])
before += br()
before += "Min weight in test intervals: "+str(self.statsBefore[2])
before += br()
document.body.add(h3("Statistics after optimization"))
after=document.body.add(div())
after+="Standard deviation in test intervals: "+str(self.statsAfter[0])
after+=br()
after+="Max weight in test intervals: "+str(self.statsAfter[1])
after+=br()
after+="Min weight in test intervals: "+str(self.statsAfter[2])
after+=br()
document.body.add(h2("Suggestion for better crontab."))
document.body.add(pre(self.newCrontab))
document.body.add(a("More information about analysis in logfile.",href="logfile.log"))
try:
with open(self._outputDir+"report.html","w") as f:
f.write(document.render())
except:
CCronLogger.error("Error while saving report.")
CCronLogger.errorToConsole("Error while saving report.")
def checkEmptyCronJobs(cronList):
newCronList=[]
badCronList=[]
for cron in cronList:
if len(cron.cronJobList) > 0:
newCronList.append(cron)
else:
CCronLogger.info("Following cron entry:"+str(cron)+" wont be analyzed or optimized, because it wont run in specified time window.")
badCronList.append(cron)
return newCronList,badCronList
def checkLength(self, length, line):
try:
dummy = int(length)
if dummy < 1:
raise ValueError
except ValueError:
CCronLogger.error("Invalid duration value in " + line)
CCronLogger.error("Setting to default: 1")
length = "1"
return length
def checkWeight(self, weight, line):
try:
dummy = int(weight)
if dummy < 1:
raise ValueError
except ValueError:
CCronLogger.error("Invalid weight value in " + line)
CCronLogger.error("Setting to default: 1")
weight = "1"
return weight
def parseCrontabs(self):
for crontab, isUser in self._crontabs:
try:
with open(crontab, "r") as file:
content = file.readlines()
self._parseCrontab(content, crontab, isUser)
except:
CCronLogger.errorToConsole(
"Following crontab file couldn't be loaded: " + crontab)
CCronLogger.error(
"Following crontab file couldn't be loaded: " + crontab)
return self._crons
def checkShifts(self, leftShift, rightShift, line):
try:
dummy = int(leftShift)
dummy2 = int(rightShift)
if dummy > dummy2:
raise ValueError
except ValueError:
CCronLogger.error("Invalid shift values in " + line)
CCronLogger.error("Setting to default: 0:0")
leftShift = "0"
rightShift = "0"
return leftShift, rightShift
def createStatistics(self,maxWeight,weights):
std=round(statistics.pstdev(weights),1)
msg="Standard deviation of weights in test intervals: "+str(std)
CCronLogger.infoToConsole(msg)
CCronLogger.info(msg)
msg="Max weight in test interval: "+str(maxWeight)
CCronLogger.infoToConsole(msg)
CCronLogger.info(msg)
return std
def checkPredictionDuration(options):
try:
duration=int(options["predictionduration"])
if duration < 1:
CCronLogger.error("Invalid duration of prediction. Setting to default: 30.")
duration=30
CCronLogger.debug("Duration of prediction: "+str(duration)+" days.")
except ValueError:
CCronLogger.error("Invalid duration of prediction. Setting to default: 30.")
duration=30
except KeyError:
CCronLogger.info("Duration of prediction directive not found. Setting to default: 30.")
duration=30
return duration
def findStartEndTime(self):
startTime=datetime.datetime.now().replace(year=9999)
endTime=datetime.datetime.now().replace(year=1)
for cron in self._cronList:
if cron.startTime<startTime:
startTime=cron.startTime
if cron.endTime>endTime:
endTime=cron.endTime
self.startTime=startTime
self.endTime=endTime
self.totalDuration=endTime-startTime
CCronLogger.debugToConsole("Start time of first job: "+str(startTime))
CCronLogger.debugToConsole("End time of last job: "+str(endTime))
CCronLogger.debugToConsole("Total duration: "+str(self.totalDuration))
def optimize(cronList, minShifts, maxShifts, sizeOfPopulation,
numberOfGenerations, crossoverProb, mutationProb, parallel,
mateOp, uniformProb, tournamentSize):
#zaregistrovani fitness
creator.create("FitnessMinLoad", base.Fitness, weights=(-1, ))
#vytvoreni jedince
creator.create("Individual", list, fitness=creator.FitnessMinLoad)
toolbox = base.Toolbox()
toolbox.register("gene", genRandomShift)
# vytvoreni listu jedincu
toolbox.register("individual", customInitRepeat, creator.Individual,
toolbox.gene, len(cronList))
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
# nastaveni fitness fce
toolbox.register("evaluate", evalOneMinLoad)
#nastaveni mate operatoru
if mateOp == "uniform":
toolbox.register("mate", tools.cxUniform, indpb=uniformProb)
elif mateOp == "onepoint":
toolbox.register("mate", tools.cxOnePoint)
elif mateOp == "twopoint":
toolbox.register("mate", tools.cxTwoPoint)
#nastaveni operatoru mutace
toolbox.register("mutate",
tools.mutUniformInt,
low=minShifts,
up=maxShifts,
indpb=1 / len(cronList))
toolbox.register("select", tools.selTournament, tournsize=tournamentSize)
pool = multiprocessing.Pool()
#vymena map za paralelni map
if parallel:
toolbox.register("map", pool.map)
else:
toolbox.register("map", map)
#random.seed(64)
random.seed(os.urandom(100))
pop = toolbox.population(n=sizeOfPopulation)
CXPB, MUTPB, NGEN = crossoverProb, mutationProb, numberOfGenerations
CCronLogger.infoToConsole("Start of evolution")
CCronLogger.info("Start of evolution")
# ohodnoceni cele populace
fitnesses = list(toolbox.map(evalOneMinLoad, pop))
for ind, fit in zip(pop, fitnesses):
ind.fitness.values = fit
CCronLogger.infoToConsole(" Evaluated %i individuals" % len(pop))
CCronLogger.info(" Evaluated %i individuals" % len(pop))
# zacatek evoluce
for g in range(NGEN):
CCronLogger.infoToConsole("-- Generation %i --" % g)
CCronLogger.info("-- Generation %i --" % g)
# vyber jedincu do dalsi populace
offspring = toolbox.select(pop, len(pop))
# klonovani vybranych jedincu
offspring = list(toolbox.map(toolbox.clone, offspring))
# aplikovani operatoru mutace a krizeni
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# ohodnoceni jedincu
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = toolbox.map(evalOneMinLoad, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
CCronLogger.infoToConsole(" Evaluated %i individuals" %
len(invalid_ind))
CCronLogger.info(" Evaluated %i individuals" % len(invalid_ind))
pop[:] = offspring
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x * x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
CCronLogger.infoToConsole(" Min %s" % min(fits))
CCronLogger.info(" Min %s" % min(fits))
CCronLogger.infoToConsole(" Max %s" % max(fits))
CCronLogger.info(" Max %s" % max(fits))
CCronLogger.infoToConsole(" Avg %s" % mean)
CCronLogger.info(" Avg %s" % mean)
CCronLogger.infoToConsole(" Std %s" % std)
CCronLogger.info(" Std %s" % std)
myMax = -1
if max(fits) > myMax:
myMax = max(fits)
CCronLogger.infoToConsole("-- End of (successful) evolution --")
CCronLogger.info("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
CCronLogger.infoToConsole("Best individual is %s, %s" %
(best_ind, best_ind.fitness.values))
CCronLogger.info("Best individual is %s, %s" %
(best_ind, best_ind.fitness.values))
cronList = applyBestShift(best_ind, cronList)
return best_ind, cronList
def setOptions(options):
try:
intervalLen = int(options["testintervallength"])
if intervalLen < 1:
CCronLogger.error(
"Invalid interval length. Setting to default: 1.")
intervalLen = 1
CCronLogger.debug("Interval length: " + str(intervalLen))
except ValueError:
CCronLogger.error("Invalid interval length. Setting to default: 1.")
intervalLen = 1
except KeyError:
CCronLogger.info(
"Interval length directive not found. Setting to default: 1.")
intervalLen = 1
try:
mateOp = options["mateoperator"]
mateOp = mateOp.lower()
if mateOp == "onepoint" or mateOp == "twopoint" or mateOp == "uniform":
CCronLogger.debug("Mate operator: " + str(mateOp))
else:
CCronLogger.error(
"Unknown value for mate operator. Setting to default: OnePoint."
)
mateOp = "onepoint"
except KeyError:
CCronLogger.info(
"Mate operator directive not found. Setting to default: OnePoint.")
mateOp = "onepoint"
try:
parallel = options["parallelcomputation"]
CCronLogger.debug("Parallel computation: " + str(parallel))
except KeyError:
CCronLogger.info(
"Parallel computation directive not found. Setting to default: false."
)
parallel = False
try:
duration = int(options["predictionduration"])
if duration < 1:
CCronLogger.error(
"Invalid duration of prediction. Setting to default: 30.")
duration = 30
CCronLogger.debug("Duration of prediction: " + str(duration) +
" days.")
except ValueError:
CCronLogger.error(
"Invalid duration of prediction. Setting to default: 30.")
duration = 30
except KeyError:
CCronLogger.info(
"Duration of prediction directive not found. Setting to default: 30."
)
duration = 30
try:
numberOfGenerations = int(options["numberofgenerations"])
if numberOfGenerations < 1:
CCronLogger.error(
"Invalid number of generations. Setting to default: 100.")
numberOfGenerations = 100
CCronLogger.debug("Number of generations: " +
str(numberOfGenerations) + " generations.")
except ValueError:
CCronLogger.error(
"Invalid number of generations. Setting to default: 100.")
numberOfGenerations = 100
except KeyError:
CCronLogger.info(
"Number of generations directive not found. Setting to default: 100."
)
numberOfGenerations = 100
try:
sizeOfPopulation = int(options["sizeofpopulation"])
if sizeOfPopulation < 1:
CCronLogger.error(
"Invalid size of populaton. Setting to default: 75.")
sizeOfPopulation = 75
CCronLogger.debug("Size of population: " + str(sizeOfPopulation) + ".")
except ValueError:
CCronLogger.error("Invalid size of populaton. Setting to default: 75.")
sizeOfPopulation = 75
except KeyError:
CCronLogger.info(
"Size of population directive not found. Setting to default: 75.")
sizeOfPopulation = 75
try:
crossoverProb = float(options["crossoverprobability"])
if crossoverProb < 0:
CCronLogger.error(
"Invalid crossover probability. Setting to default: 0.75.")
crossoverProb = 0.75
CCronLogger.debug("Crossover probability: " +
str(crossoverProb * 100) + " %.")
except ValueError:
CCronLogger.error(
"Invalid crossover probability. Setting to default: 0.75.")
crossoverProb = 0.75
except KeyError:
CCronLogger.info(
"Invalid probability directive not found. Setting to default: 0.75."
)
crossoverProb = 0.75
try:
mutationProb = float(options["mutationprobability"])
if mutationProb < 0:
CCronLogger.error(
"Invalid mutation probability. Setting to default: 0.05.")
mutationProb = 0.05
CCronLogger.debug("Mutation probability: " + str(mutationProb * 100) +
" %.")
except ValueError:
CCronLogger.error(
"Invalid mutation probability. Setting to default: 0.05.")
mutationProb = 0.05
except KeyError:
CCronLogger.info(
"Mutation probability directive not found. Setting to default: 0.05."
)
mutationProb = 0.05
try:
uniformProb = float(options["uniformoperatorprobability"])
if uniformProb < 0:
CCronLogger.error(
"Invalid Uniform crossover probability. Setting to default: 0.75."
)
uniformProb = 0.75
CCronLogger.debug("Uniform crossover probability: " +
str(uniformProb * 100) + " %.")
except ValueError:
CCronLogger.error(
"Invalid Uniform crossover probability. Setting to default: 0.75.")
uniformProb = 0.75
except KeyError:
CCronLogger.info(
"Uniform crossover probability directive not found. Setting to default: 0.75."
)
uniformProb = 0.75
try:
tournamentSize = int(options["tournamentsize"])
if tournamentSize < 2:
CCronLogger.error(
"Invalid tournament size. Setting to default: 3.")
tournamentSize = 3
CCronLogger.debug("Tournament size: " + str(tournamentSize) + ".")
except ValueError:
CCronLogger.error("Invalid tournament size. Setting to default: 3.")
tournamentSize = 3
except KeyError:
CCronLogger.info(
"Tournament size directive not found. Setting to default: 3.")
tournamentSize = 3
return intervalLen, duration, numberOfGenerations, sizeOfPopulation, crossoverProb, mutationProb, parallel, mateOp, uniformProb, tournamentSize
def setOptions(self, options):
try:
self._intervals = int(options["intervalsperday"])
CCronLogger.debug("Intervals per day: " + str(self._intervals))
except ValueError:
CCronLogger.error(
"Invalid intervals per day. Setting to default: 10.")
self._intervals = 10
except KeyError:
CCronLogger.info(
"Intervals per day directive not found. Setting to default: 10."
)
self._intervals = 10
try:
self._duration = int(options["predictionduration"])
CCronLogger.debug("Duration of prediction: " +
str(self._duration) + " days.")
except ValueError:
CCronLogger.error(
"Invalid duration of prediction. Setting to default: 30.")
self._duration = 30
except KeyError:
CCronLogger.info(
"Duration of prediction directive not found. Setting to default: 30."
)
self._duration = 30
try:
self._numberOfGenerations = int(options["numberofgenerations"])
CCronLogger.debug("Number of generations: " +
str(self._numberOfGenerations) + " generations.")
except ValueError:
CCronLogger.error(
"Invalid number of generations. Setting to default: 100.")
self._numberOfGenerations = 100
except KeyError:
CCronLogger.info(
"Number of generations directive not found. Setting to default: 100."
)
self._numberOfGenerations = 100
try:
self._sizeOfPopulation = int(options["sizeofpopulation"])
CCronLogger.debug("Size of population: " +
str(self._sizeOfPopulation) + ".")
except ValueError:
CCronLogger.error(
"Invalid size of populaton. Setting to default: 75.")
self._sizeOfPopulation = 75
except KeyError:
CCronLogger.info(
"Size of population directive not found. Setting to default: 75."
)
self._sizeOfPopulation = 75
try:
self._crossoverProb = float(options["crossoverprobability"])
CCronLogger.debug("Crossover probability: " +
str(self._crossoverProb * 100) + " %.")
except ValueError:
CCronLogger.error(
"Invalid crossover probability. Setting to default: 0.75.")
self._crossoverProb = 0.75
except KeyError:
CCronLogger.info(
"Invalid probability directive not found. Setting to default: 0.75."
)
self._crossoverProb = 0.75
try:
self._mutationProb = float(options["mutationprobability"])
CCronLogger.debug("Mutation probability: " +
str(self._mutationProb * 100) + " %.")
except ValueError:
CCronLogger.error(
"Invalid mutation probability. Setting to default: 0.05.")
self._mutationProb = 0.05
except KeyError:
CCronLogger.info(
"Mutation probability directive not found. Setting to default: 0.05."
)
self._mutationProb = 0.05
def handler(signum, frame):
CCronLogger.infoToConsole("Closing program.")
exit(0)
from CronLogger import CCronLogger
try:
import dominate
from dominate.tags import *
except ImportError:
CCronLogger.errorToConsole("You don't have required Python module.")
CCronLogger.debugToConsole("Required modules are: dominate")
class CCronReport:
def __init__(self,options, newCrontab, statsBefore, statsAfter, shiftVector, cronList):
self.setOptions(options)
self.newCrontab=newCrontab
self.statsBefore=statsBefore
self.statsAfter=statsAfter
self.shiftVector=shiftVector
self.cronList=cronList
def setOptions(self,options):
try:
self._outputDir=options["outputdir"]
CCronLogger.debug("Output directory: "+self._outputDir)
except KeyError:
CCronLogger.info("Output directory directive not found. Setting to default: working directory.")
self._outputDir="./"
try:
self._graphName=options["graphname"]
CCronLogger.debug("Graph name: "+self._graphName)
except KeyError:
CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.")
def setOptions(self,options):
try:
self._outputDir=options["outputdir"]
CCronLogger.debug("Output directory: "+self._outputDir)
except KeyError:
CCronLogger.info("Output directory directive not found. Setting to default: working directory.")
self._outputDir="./"
try:
self._graphName=options["graphname"]
CCronLogger.debug("Graph name: "+self._graphName)
except KeyError:
CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.")
self._graphName="graph.pdf"
try:
self._weightGraphName=options["weightgraphname"]
CCronLogger.debug("Weight graph name: "+self._weightGraphName)
except KeyError:
CCronLogger.info("Weight graph name directive not found. Setting to default: weightGraph.pdf.")
self._weightGraphName="weightGraph.pdf"
def setOptions(self,options):
try:
self._threshold=int(options["threshold"])
if self._threshold < 0:
CCronLogger.error("Invalid threshold. Setting to default: 30.")
self._threshold=30
CCronLogger.debug("Threshold: "+str(self._threshold))
except ValueError:
CCronLogger.error("Invalid threshold. Setting to default: 30.")
self._threshold=30
except KeyError:
CCronLogger.info("Threshold directive not found. Setting to default: 30.")
self._threshold=30
try:
self._duration=int(options["predictionduration"])
if self._duration < 1:
CCronLogger.error("Invalid duration of prediction. Setting to default: 30.")
self._duration=30
CCronLogger.debug("Duration of prediction: "+str(self._duration)+" days.")
except ValueError:
CCronLogger.error("Invalid duration of prediction. Setting to default: 30.")
self._duration=30
except KeyError:
CCronLogger.info("Duration of prediction directive not found. Setting to default: 30.")
self._duration=30
try:
self._intervalLen=int(options["testintervallength"])
if self._intervalLen < 1:
CCronLogger.error("Invalid interval length. Setting to default: 1.")
self._intervalLen=1
CCronLogger.debug("Interval length: "+str(self._intervalLen))
except ValueError:
CCronLogger.error("Invalid interval length. Setting to default: 1.")
self._intervalLen=1
except KeyError:
CCronLogger.info("Interval length directive not found. Setting to default: 1.")
self._intervalLen=1
try:
self._samplesize=int(options["loadgraphsamplesize"])
if self._samplesize < 1:
CCronLogger.error("Invalid sample size. Setting to default: 200.")
self._samplesize=200
CCronLogger.debug("Sample size: "+str(self._samplesize))
except ValueError:
CCronLogger.error("Invalid sample size. Setting to default: 200.")
self._samplesize=200
except KeyError:
CCronLogger.info("Sample size directive not found. Setting to default: 200.")
self._samplesize=200
try:
self._cronLenLimit=int(options["ignoreshortcronjobslimit"])
if self._cronLenLimit < 1:
CCronLogger.error("Invalid short cron job limit. Setting to default: 30s.")
self._cronLenLimit=30
CCronLogger.debug("Short cron job limit: "+str(self._cronLenLimit)+" seconds.")
except ValueError:
CCronLogger.error("Invalid short cron job limit. Setting to default: 30s.")
self._cronLenLimit=30
except KeyError:
CCronLogger.info("Short cron job limit directive not found. Setting to default: 30s.")
self._cronLenLimit=30
try:
self._showImages=options["showinteractiveimages"]
CCronLogger.debug("Show interactive image: "+str(self._showImages))
except KeyError:
CCronLogger.info("Show interactive images directive not found. Setting to default: True.")
self._showImages=True
try:
self._ignoreShort=options["ignoreshortcronjobs"]
CCronLogger.debug("Ignore short cron jobs: "+str(self._ignoreShort))
except KeyError:
CCronLogger.info("Ignore short cron jobs directive not found. Setting to default: False.")
self._ignoreShort=False
try:
self._outputDir=options["outputdir"]
CCronLogger.debug("Output directory: "+self._outputDir)
except KeyError:
CCronLogger.info("Output directory directive not found. Setting to default: working directory.")
self._outputDir="./"
try:
self._timeFormat=options["timeformatonxaxis"]
try:
_=datetime.datetime.now().strftime(self._timeFormat)
except ValueError:
CCronLogger.error("Invalid time format. Setting to default: %d.%m.%Y %H:%M:%S")
self._timeFormat="%d.%m.%Y %H:%M:%S"
CCronLogger.debug("Time format on X axis: "+self._timeFormat)
except KeyError:
CCronLogger.info("Time format on X axis directive not found. Setting to default: %d.%m.%Y %H:%M:%S")
self._timeFormat="%d.%m.%Y %H:%M:%S"
try:
self.weightGraphLineWidth=float(options["weightgraphlinewidth"])
if self.weightGraphLineWidth < 0.5:
CCronLogger.error("Invalid weight graph line width. Setting to default: 0.75.")
self.weightGraphLineWidth=0.75
CCronLogger.debug("Weight graph line width: "+str(self.weightGraphLineWidth))
except ValueError:
CCronLogger.error("Invalid weight graph line width. Setting to default: 0.75.")
self.weightGraphLineWidth=0.75
except KeyError:
CCronLogger.info("Weight graph line width directive not found. Setting to default: 0.75.")
self.weightGraphLineWidth=0.75
try:
self._graphName=options["graphname"]
CCronLogger.debug("Graph name: "+self._graphName)
except KeyError:
CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.")
self._graphName="graph.pdf"
try:
self._weightGraphName=options["weightgraphname"]
CCronLogger.debug("Weight graph name: "+self._weightGraphName)
except KeyError:
CCronLogger.info("Weight graph name directive not found. Setting to default: weightGraph.pdf.")
self._weightGraphName="weightGraph.pdf"
try:
self._approximation=options["loadgraphapproximation"]
CCronLogger.debug("Weight graph approximaton: "+str(self._approximation))
except KeyError:
CCronLogger.info("Weight graph directive not found. Setting to default: True.")
self._approximation=True
import datetime
from datetime import timedelta
from collections import OrderedDict
from CronLogger import CCronLogger
import random
import statistics
try:
import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter
from matplotlib.pyplot import savefig
import intervaltree
from numpy import asarray
except ImportError:
CCronLogger.errorToConsole("You don't have required Python module.")
CCronLogger.debugToConsole("Required modules are: matplotlib, intervaltree, numpy")
class CCronPlotter:
def __init__(self,cronList,options,plotBetter=False):
self.plotBetter=plotBetter
self.setOptions(options)
self._endPoints=dict()
self._weights=[]
self._cronList=cronList
self.oneMinute=timedelta(minutes=1)
self.oneHour=timedelta(hours=1)
self.oneDay=timedelta(days=1)
self.oneWeek=timedelta(weeks=1)
self.oneMonth=timedelta(weeks=4)
def __init__(self, options):
self.setOptions(options)
self._crontabFolders = []
#pridat slozky s crontaby do seznamu
#obecna slozka s crontaby
if self.includeCrondFolder:
self._crontabFolders.append(["/etc/cron.d/", False])
#uzivatelske slozky s crontaby
if os.geteuid() == 0:
for usercrontabdir in self.userCrontabDirs.split(":"):
self._crontabFolders.append([usercrontabdir, True])
else:
CCronLogger.info(
"You have to run this tool as root to access user crontabs.")
CCronLogger.infoToConsole(
"You have to run this tool as root to access user crontabs.")
#ostatni slozky s crontaby, ktere nejsou uzivatelske
for othercrontabdir in self.otherCrontabDirs.split(":"):
if othercrontabdir != "":
self._crontabFolders.append([othercrontabdir, False])
#muj testovaci crontab
#self._crontabFolders.append(["/home/petr/git/cronalyzer/data/crontabs/",True])
self._crontabs = []
for crontabFolder, isUser in self._crontabFolders:
for crontab in glob.glob(os.path.join(crontabFolder, "*")):
if not crontab.endswith("~") and os.path.isfile(crontab):
self._crontabs.append([crontab, isUser])
CCronLogger.info("Added crontab: " + crontab +
" to analysis.")
CCronLogger.infoToConsole("Added crontab: " + crontab +
" to analysis.")
# + /etc/crontab
if self.includeMainCrontab:
self._crontabs.append(["/etc/crontab", False])
CCronLogger.info("Added crontab: /etc/crontab to analysis.")
CCronLogger.infoToConsole(
"Added crontab: /etc/crontab to analysis.")
#list s rozparsovanymi crony
self._crons = []
def _parseCrontab(self, content, sourceCrontab, isUser):
CCronLogger.infoToConsole("Parsing crontabs...")
CCronLogger.info("Parsing crontabs...")
lineNr = 0
for line in content:
#odstraneni bilych znaku z konce radky
line = line.rstrip()
#pokud je to radek zacinajici # = komentar nebo prazdna radka, tak ji preskoc
if re.match("^[ \t]*#", line) or line == "":
continue
#REGEX na radku s cron ulohou
#cronEntry=re.match("^[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^#\n]*)[ \t]*#?[ \t]*([^\n]*)$", line)
cronEntry = line.split()
#neni to crontab zaznam, pravdepodovne nastaveni promenne prostredi
if (isUser and len(cronEntry) < 6) or (not isUser
and len(cronEntry) < 7):
continue
#minute=cronEntry.group(1)
#hour=cronEntry.group(2)
#day=cronEntry.group(3)
#month=cronEntry.group(4)
#dayOfWeek=cronEntry.group(5)
#command=cronEntry.group(6)
#comment=cronEntry.group(7)
minute = cronEntry[0]
hour = cronEntry[1]
day = cronEntry[2]
month = cronEntry[3]
dayOfWeek = cronEntry[4]
if not self._checkMinute(minute) or not self._checkHour(
hour) or not self._checkDay(day) or not self._checkMonth(
month) or not self._checkDayOfWeek(dayOfWeek):
msg = "Error while parsing crontab entry in file: " + sourceCrontab
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
if isUser:
commandAndComment = cronEntry[5:]
user = ""
else:
user = cronEntry[5]
commandAndComment = cronEntry[6:]
comment = []
command = []
commentStarted = False
jednoducheUv = 0
dvojiteUv = 0
for word in commandAndComment:
jednoducheUv += word.count("'")
dvojiteUv += word.count('"')
if re.search(
"#",
word) and not dvojiteUv % 2 and not jednoducheUv % 2:
commentStarted = True
if commentStarted:
word = word.replace("#", "")
words = word.split(",")
for word in words:
if word != "":
comment.append(word)
else:
command.append(word)
#check na neukoncene uvozovky
if jednoducheUv % 2 or dvojiteUv % 2:
msg = "Unfinished quotation marks, skipping cron entry"
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
command = " ".join(command)
length = self._getMetadata(comment, "duration", lineNr,
sourceCrontab)
weight = self._getMetadata(comment, "weight", lineNr,
sourceCrontab)
shift = self._getMetadata(comment, "shift", lineNr, sourceCrontab)
if length is None:
length = self.defaultLength
if weight is None:
weight = self.defaultWeight
if shift is None:
shift = str(self.defaultMinShift) + ":" + str(
self.defaultMaxShift)
if shift is not None:
shifts = shift.split(":")
if len(shifts) < 2:
leftShift = "-" + shift
rightShift = shift
else:
leftShift = shifts[0]
rightShift = shifts[1]
leftShift, rightShift = self.checkShifts(leftShift, rightShift,
line)
weight = self.checkWeight(weight, line)
length = self.checkLength(length, line)
try:
cron = CCron(minute, hour, day, month, dayOfWeek, user,
command, int(length), int(weight), int(leftShift),
int(rightShift), sourceCrontab,
self.includeRareJobs)
self._crons.append(cron)
except ValueError:
errMsg = "Error occured while predicting following cronjob: \n"
errMsg = errMsg + minute + " " + hour + " " + day + " " + month + " " + dayOfWeek + " " + user + " " + command + "\n"
errMsg = errMsg + "from crontab: " + sourceCrontab + "\n"
errMsg = errMsg + "maybe job is badly configured or check \"include rare cron job\" directive"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
lineNr += 1
def plotWeightGraph(self, length,centers, weights, maxWeight):
#inicializace
width = int(self._duration*10)
if width > 25:
width=25
fig = plt.figure(figsize=(width ,6))
ax = fig.add_subplot(111)
thresholds=asarray([self._threshold]*len(weights))
plt.plot(centers,weights,color="k",lw=self.weightGraphLineWidth)
plt.fill_between(centers,weights,thresholds,where=weights>thresholds,interpolate=True,color="r")
ax.xaxis_date()
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
#popis osyX
plt.xlabel('Time')
plt.ylabel("Load")
fig.autofmt_xdate()
padd=maxWeight/10
plt.ylim(0,maxWeight+padd)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
try:
savefig(self._outputDir+"improved_"+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
try:
savefig(self._outputDir+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()
def setOptions(self, options):
try:
self.defaultLength = int(options["defaultduration"])
if self.defaultLength < 1:
CCronLogger.error(
"Invalid default duration. Setting to default: 1.")
self.defaultLength = 1
CCronLogger.debug("Default weight: " + str(self.defaultLength) +
".")
except ValueError:
CCronLogger.error(
"Invalid default duration. Setting to default: 1.")
self.defaultLength = 1
except KeyError:
CCronLogger.info(
"Default duration directive not found. Setting to default: 1.")
self.defaultLength = 1
try:
self.defaultWeight = int(options["defaultweight"])
if self.defaultWeight < 1:
CCronLogger.error(
"Invalid default weight. Setting to default: 5.")
self.defaultWeight = 5
CCronLogger.debug("Default weight: " + str(self.defaultWeight) +
".")
except ValueError:
CCronLogger.error("Invalid default weight. Setting to default: 5.")
self.defaultWeight = 5
except KeyError:
CCronLogger.info(
"Default weight directive not found. Setting to default: 5.")
self.defaultWeight = 5
try:
self.defaultMaxShift = int(options["defaultmaxshift"])
CCronLogger.debug("Default MinShift: " +
str(self.defaultMaxShift) + ".")
except ValueError:
CCronLogger.error(
"Invalid default MaxShift. Setting to default: 0.")
self.defaultMaxShift = 0
except KeyError:
CCronLogger.info(
"Default MaxShift directive not found. Setting to default: 0.")
self.defaultMaxShift = 0
try:
self.defaultMinShift = int(options["defaultminshift"])
CCronLogger.debug("Default MinShift: " +
str(self.defaultMinShift) + ".")
except ValueError:
CCronLogger.error(
"Invalid default MinShift. Setting to default: 0.")
self.defaultMinShift = 0
except KeyError:
CCronLogger.info(
"Default MinShift directive not found. Setting to default: 0.")
self.defaultMinShift = 0
if self.defaultMinShift > self.defaultMaxShift:
CCronLogger.error(
"Invalid default MinShift and MaxShift. Setting both 0.")
self.defaultMaxShift = 0
self.defaultMinShift = 0
try:
self.userCrontabDirs = options["usercrontabdirs"]
CCronLogger.debug("User crontab directories: " +
str(self.userCrontabDirs))
except KeyError:
CCronLogger.info("User crontab directories directive not found.")
self.userCrontabDirs = ""
try:
self.otherCrontabDirs = options["othercrontabdirs"]
CCronLogger.debug("Other crontab directories: " +
str(self.otherCrontabDirs))
except KeyError:
CCronLogger.info("Other crontab directories directive not found.")
self.otherCrontabDirs = ""
try:
self.includeRareJobs = options["includerarecronjobs"]
CCronLogger.debug("Include rare cron jobs: " +
str(self.includeRareJobs))
except KeyError:
CCronLogger.info(
"Include rare cron jobs directive not found. Setting to default: True."
)
self.includeRareJobs = True
try:
self.includeMainCrontab = options["includemaincrontab"]
CCronLogger.debug("Include main crontab: " +
str(self.includeMainCrontab))
except KeyError:
CCronLogger.info(
"Include main crontab directive not found. Setting to default: False."
)
self.includeMainCrontab = False
try:
self.includeCrondFolder = options["includecrondfolder"]
CCronLogger.debug("Include /etc/cron.d/ folder: " +
str(self.includeCrondFolder))
except KeyError:
CCronLogger.info(
"Include /etc/cron.d/ folder directive not found. Setting to default: False."
)
self.includeCrondFolder = False
def plotCronJobs(self):
if len(self._cronList) == 0:
CCronLogger.errorToConsole("No viable cron jobs to plot and analyze.")
return False,None,None,None
height_graph=2+ (len(self._cronList) / 2)
#inicializace
fig = plt.figure(figsize=(8,height_graph))
ax = fig.add_subplot(111)
self._countTotalOverlaps()
self._setLineWidth()
firstTime=datetime.datetime.now().replace(year=9999)
lastTime=datetime.datetime.now().replace(year=1)
cronNameList=[]
yLabels=[]
cronCounter=0
cronID=1
counter=0
shown=False
print("Plotting cron jobs.")
for cron in self._cronList:
if self._ignoreShort and cron.duration < timedelta(seconds=self._cronLenLimit):
CCronLogger.debug("Cron job "+str(cron)+" is too short, it will be skipped.")
continue
#pokud je prikaz moc dlouhy, orizni ho
if len(cron.command) > 30:
cronNameList.append(cron.command[0:27]+" ...")
else:
cronNameList.append(cron.command)
if cron.overlaps > 0:
CCronLogger.info("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
CCronLogger.infoToConsole("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
yLabels.append(cronID + (cron.overlaps / 2) )
jobID=0
for job in cron.cronJobList:
self._weights.append([job.startTime,job.endTime,cron.weight])
self._endPoints[job.startTime]=None
self._endPoints[job.endTime]=None
if job.startTime<firstTime:
firstTime=job.startTime
if job.endTime>lastTime:
lastTime=job.endTime
#cron joby se prekryvaji
if cron.overlaps>0:
#vykresli je nad sebe, cervene
self._drawLine(cronID + (jobID % (cron.overlaps + 1) ),job.startTime,job.endTime,"r")
#crony se neprekryvaji
else:
self._drawLine(cronID,job.startTime,job.endTime,"k")
#prubeh
progress=int(counter*100/self.cronJobsTotal)
if not progress%5:
if not shown:
CCronLogger.debugToConsoleWithLineFeed("Progress: "+str(progress)+"%")
shown=True
else:
shown=False
counter+=1
jobID+=1
cronID+=(cron.overlaps+1)
cronCounter+=1
#pokud to neni posledni vykreslovany cron, udelej vodorovnou caru
if cronCounter!=len(self._cronList):
plt.axhline(cronID - 0.5,color="black")
CCronLogger.debugToConsole("Progress: 100%")
if counter == 0:
CCronLogger.error("No crons to plot.")
CCronLogger.errorToConsole("No crons to plot.")
return False,None,None,None
#osaX = cas
ax.xaxis_date()
#format casu na ose
length=lastTime-firstTime
self.startTime=firstTime
self.endTime=lastTime
self.duration=length
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
delta=(lastTime-firstTime)/100
#rozsah osy y
plt.ylim(-0.5,cronID)
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
plt.xlim(firstTime-delta, lastTime+delta)
#popis osyX
plt.xlabel('Time')
#plt.ylabel("Cron jobs")
fig.autofmt_xdate()
centers, weights, maxWeight, minWeight, summedWeights,weightsForStatistics = self.makeWeightIntervals()
std=self.createStatistics(maxWeight, weightsForStatistics)
loadPeaks = self.findLoadPeaks(summedWeights)
if len(loadPeaks) > 0:
yLabels.insert(0, 0)
cronNameList.insert(0, "Bottlenecks")
plt.axhline(0.5,color="black",lw=2.5)
loadPeaksLen=timedelta(seconds=0)
totalWeight=0
for peak in loadPeaks:
plt.hlines(0, peak[0], peak[1], "r", lw=self.lineWidth)
startTime=peak[0].strftime("%d.%m.%Y %H:%M:%S")
endTime=peak[1].strftime("%d.%m.%Y %H:%M:%S")
totalWeight+= peak[2]
loadPeaksLen+=(peak[1]-peak[0])
CCronLogger.info("Found bottleneck: "+startTime+" <---> "+endTime)
CCronLogger.info("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
CCronLogger.infoToConsole("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
#popisky osyY: 1.arg list ID (unikatni cronjoby), 2.arg = list popisku (nazvy cronjobu)
plt.yticks(yLabels, cronNameList)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
try:
savefig(self._outputDir+"improved_"+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
try:
savefig(self._outputDir+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()
self.plotWeightGraph(length, centers, weights, maxWeight)
return True,std,maxWeight,minWeight
def main():
#pokud neni zadan skript, zkusi se doplnit z aktualniho adresare
if len(sys.argv[1:]) == 0:
configFile = "config.conf"
else:
configFile = readArguments(sys.argv[1:])
testConfigFile(configFile)
configParser = CConfigParser(configFile)
options = configParser.parseConfig()
#nacteni a vytvoreni slozky
try:
outputDir = options["outputdir"]
except KeyError:
outputDir = "."
options["outputdir"] = repairOutputDir(outputDir)
createOutputDir(outputDir)
logFile = getLogFile(options)
checkLogFile(logFile)
try:
logLevel = options["loglevel"]
except KeyError:
logFile = "info"
try:
removeLog = options["removelogonstart"]
except KeyError:
removeLog = False
try:
onlyAnalysis = options["onlyanalysis"]
except KeyError:
onlyAnalysis = False
CCronLogger.logFile = logFile
CCronLogger.logLevel = CCronLogger.setLogLevel(logLevel)
CCronLogger.removeLog = removeLog
CCronLogger.initLog()
#parsovani crontabu
cronParser = CCronParser(options)
cronList = cronParser.parseCrontabs()
crontabs = cronParser.getCrontabs()
predictionDuration = checkPredictionDuration(options)
timeInterval = timedelta(days=predictionDuration)
now = datetime.datetime.now()
then = now + timeInterval
#predikce uloh
i = 0
for cron in cronList:
try:
cron.predictUntil(then)
except ValueError:
errMsg = "Error occured while predicting following cronjob " + str(
cron) + ", maybe job is badly configured"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
#cron.printCron()
i += 1
#zkontroluje prazdne cron joby
cronList, emptyCronList = checkEmptyCronJobs(cronList)
#vykreslovani uloh
plotter = CCronPlotter(cronList, options)
anyCrons, std, maxW, minW = plotter.plotCronJobs()
if not anyCrons:
exit(1)
statsBefore = [std, maxW, minW]
start = plotter.startTime
end = plotter.endTime
duration = plotter.duration
if onlyAnalysis:
exit(0)
#zavolani optimalizatoru
shiftVector, newCronList = optimize.main(cronList, options)
CCronLogger.info("Starting to plot improved cron jobs.")
CCronLogger.infoToConsole("Starting to plot improved cron jobs.")
#vykresleni uloh po optimalizaci
plotterAfter = CCronPlotter(newCronList, options, True)
anyCrons, std, maxW, minW = plotterAfter.plotCronJobs()
statsAfter = [std, maxW, minW]
#vytvoreni noveho crontabu
crontabCreator = CCrontabCreator(shiftVector, cronList, emptyCronList,
crontabs)
newCrontab = crontabCreator.createCrontab()
reporter = CCronReport(options, newCrontab, statsBefore, statsAfter,
shiftVector, cronList)
reporter.createReport()
CCronLogger.finishLog(statsBefore, statsAfter)
#!/usr/bin/python3
import random
from datetime import timedelta
import datetime
from CronJob import CCronJob
from CronLogger import CCronLogger
import statistics
import multiprocessing
import os
try:
from deap import creator, base, tools
import intervaltree
except ImportError:
CCronLogger.errorToConsole("You don't have required Python module.")
CCronLogger.debugToConsole("Required modules are: deap, intervaltree")
def init(cronList, options):
global cnt
cnt = 0
global intervalLen
intervalLen, duration, numberOfGenerations, sizeOfPopulation, crossoverProb, mutationProb, parallel, mateOp, uniformProb, tournamentSize = setOptions(
options)
origWeights2, minShifts2, maxShifts2 = prepareWeightList(cronList)
global origWeights