def createReport(self):
document = dominate.document(title="Ouput report from Cronalyzer")
document.head.add(meta(charset="utf-8"))
document.body.add(h1("Output report from Cronalyzer"))
document.body.add(h2("Plot with cronjobs before optimization:"))
document.body.add(img(src=self._graphName, alt="Plot with cronjobs."))
document.body.add(h2("Plot with weights of cronjobs before optimization:"))
document.body.add(img(src=self._weightGraphName, alt="Plot with weights of cronjobs."))
document.body.add(h2("Plot with cronjobs after optimization:"))
document.body.add(img(src="improved_"+self._graphName, alt="Plot with cronjobs after optimization."))
document.body.add(h2("Plot with weights of cronjobs after optimization:"))
document.body.add(img(src="improved_"+self._weightGraphName, alt="Plot with weights of cronjobs after optimization."))
document.body.add(h2("List of jobs and recommended shift in minutes"))
vector=document.body.add(div())
cronID=0
for cron in self.cronList:
shiftAmount=self.shiftVector[cronID]
vector += str(cron)+" should be shift by: "+str(shiftAmount)+" minutes."
vector += br()
cronID+=1
document.body.add(h2("Summary of optimization"))
document.body.add(h3("Statistics before optimization"))
before=document.body.add(div())
before += "Standard deviation in test intervals: "+str(self.statsBefore[0])
before += br()
before += "Max weight in test intervals: "+str(self.statsBefore[1])
before += br()
before += "Min weight in test intervals: "+str(self.statsBefore[2])
before += br()
document.body.add(h3("Statistics after optimization"))
after=document.body.add(div())
after+="Standard deviation in test intervals: "+str(self.statsAfter[0])
after+=br()
after+="Max weight in test intervals: "+str(self.statsAfter[1])
after+=br()
after+="Min weight in test intervals: "+str(self.statsAfter[2])
after+=br()
document.body.add(h2("Suggestion for better crontab."))
document.body.add(pre(self.newCrontab))
document.body.add(a("More information about analysis in logfile.",href="logfile.log"))
try:
with open(self._outputDir+"report.html","w") as f:
f.write(document.render())
except:
CCronLogger.error("Error while saving report.")
CCronLogger.errorToConsole("Error while saving report.")
def parseCrontabs(self):
for crontab, isUser in self._crontabs:
try:
with open(crontab, "r") as file:
content = file.readlines()
self._parseCrontab(content, crontab, isUser)
except:
CCronLogger.errorToConsole(
"Following crontab file couldn't be loaded: " + crontab)
CCronLogger.error(
"Following crontab file couldn't be loaded: " + crontab)
return self._crons
def main():
#pokud neni zadan skript, zkusi se doplnit z aktualniho adresare
if len(sys.argv[1:]) == 0:
configFile = "config.conf"
else:
configFile = readArguments(sys.argv[1:])
testConfigFile(configFile)
configParser = CConfigParser(configFile)
options = configParser.parseConfig()
#nacteni a vytvoreni slozky
try:
outputDir = options["outputdir"]
except KeyError:
outputDir = "."
options["outputdir"] = repairOutputDir(outputDir)
createOutputDir(outputDir)
logFile = getLogFile(options)
checkLogFile(logFile)
try:
logLevel = options["loglevel"]
except KeyError:
logFile = "info"
try:
removeLog = options["removelogonstart"]
except KeyError:
removeLog = False
try:
onlyAnalysis = options["onlyanalysis"]
except KeyError:
onlyAnalysis = False
CCronLogger.logFile = logFile
CCronLogger.logLevel = CCronLogger.setLogLevel(logLevel)
CCronLogger.removeLog = removeLog
CCronLogger.initLog()
#parsovani crontabu
cronParser = CCronParser(options)
cronList = cronParser.parseCrontabs()
crontabs = cronParser.getCrontabs()
predictionDuration = checkPredictionDuration(options)
timeInterval = timedelta(days=predictionDuration)
now = datetime.datetime.now()
then = now + timeInterval
#predikce uloh
i = 0
for cron in cronList:
try:
cron.predictUntil(then)
except ValueError:
errMsg = "Error occured while predicting following cronjob " + str(
cron) + ", maybe job is badly configured"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
#cron.printCron()
i += 1
#zkontroluje prazdne cron joby
cronList, emptyCronList = checkEmptyCronJobs(cronList)
#vykreslovani uloh
plotter = CCronPlotter(cronList, options)
anyCrons, std, maxW, minW = plotter.plotCronJobs()
if not anyCrons:
exit(1)
statsBefore = [std, maxW, minW]
start = plotter.startTime
end = plotter.endTime
duration = plotter.duration
if onlyAnalysis:
exit(0)
#zavolani optimalizatoru
shiftVector, newCronList = optimize.main(cronList, options)
CCronLogger.info("Starting to plot improved cron jobs.")
CCronLogger.infoToConsole("Starting to plot improved cron jobs.")
#vykresleni uloh po optimalizaci
plotterAfter = CCronPlotter(newCronList, options, True)
anyCrons, std, maxW, minW = plotterAfter.plotCronJobs()
statsAfter = [std, maxW, minW]
#vytvoreni noveho crontabu
crontabCreator = CCrontabCreator(shiftVector, cronList, emptyCronList,
crontabs)
newCrontab = crontabCreator.createCrontab()
reporter = CCronReport(options, newCrontab, statsBefore, statsAfter,
shiftVector, cronList)
reporter.createReport()
CCronLogger.finishLog(statsBefore, statsAfter)
def _parseCrontab(self, content, sourceCrontab, isUser):
CCronLogger.infoToConsole("Parsing crontabs...")
CCronLogger.info("Parsing crontabs...")
lineNr = 0
for line in content:
#odstraneni bilych znaku z konce radky
line = line.rstrip()
#pokud je to radek zacinajici # = komentar nebo prazdna radka, tak ji preskoc
if re.match("^[ \t]*#", line) or line == "":
continue
#REGEX na radku s cron ulohou
#cronEntry=re.match("^[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^\s]+)[ \t]*([^#\n]*)[ \t]*#?[ \t]*([^\n]*)$", line)
cronEntry = line.split()
#neni to crontab zaznam, pravdepodovne nastaveni promenne prostredi
if (isUser and len(cronEntry) < 6) or (not isUser
and len(cronEntry) < 7):
continue
#minute=cronEntry.group(1)
#hour=cronEntry.group(2)
#day=cronEntry.group(3)
#month=cronEntry.group(4)
#dayOfWeek=cronEntry.group(5)
#command=cronEntry.group(6)
#comment=cronEntry.group(7)
minute = cronEntry[0]
hour = cronEntry[1]
day = cronEntry[2]
month = cronEntry[3]
dayOfWeek = cronEntry[4]
if not self._checkMinute(minute) or not self._checkHour(
hour) or not self._checkDay(day) or not self._checkMonth(
month) or not self._checkDayOfWeek(dayOfWeek):
msg = "Error while parsing crontab entry in file: " + sourceCrontab
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
if isUser:
commandAndComment = cronEntry[5:]
user = ""
else:
user = cronEntry[5]
commandAndComment = cronEntry[6:]
comment = []
command = []
commentStarted = False
jednoducheUv = 0
dvojiteUv = 0
for word in commandAndComment:
jednoducheUv += word.count("'")
dvojiteUv += word.count('"')
if re.search(
"#",
word) and not dvojiteUv % 2 and not jednoducheUv % 2:
commentStarted = True
if commentStarted:
word = word.replace("#", "")
words = word.split(",")
for word in words:
if word != "":
comment.append(word)
else:
command.append(word)
#check na neukoncene uvozovky
if jednoducheUv % 2 or dvojiteUv % 2:
msg = "Unfinished quotation marks, skipping cron entry"
CCronLogger.errorToConsole(msg)
CCronLogger.error(msg)
continue
command = " ".join(command)
length = self._getMetadata(comment, "duration", lineNr,
sourceCrontab)
weight = self._getMetadata(comment, "weight", lineNr,
sourceCrontab)
shift = self._getMetadata(comment, "shift", lineNr, sourceCrontab)
if length is None:
length = self.defaultLength
if weight is None:
weight = self.defaultWeight
if shift is None:
shift = str(self.defaultMinShift) + ":" + str(
self.defaultMaxShift)
if shift is not None:
shifts = shift.split(":")
if len(shifts) < 2:
leftShift = "-" + shift
rightShift = shift
else:
leftShift = shifts[0]
rightShift = shifts[1]
leftShift, rightShift = self.checkShifts(leftShift, rightShift,
line)
weight = self.checkWeight(weight, line)
length = self.checkLength(length, line)
try:
cron = CCron(minute, hour, day, month, dayOfWeek, user,
command, int(length), int(weight), int(leftShift),
int(rightShift), sourceCrontab,
self.includeRareJobs)
self._crons.append(cron)
except ValueError:
errMsg = "Error occured while predicting following cronjob: \n"
errMsg = errMsg + minute + " " + hour + " " + day + " " + month + " " + dayOfWeek + " " + user + " " + command + "\n"
errMsg = errMsg + "from crontab: " + sourceCrontab + "\n"
errMsg = errMsg + "maybe job is badly configured or check \"include rare cron job\" directive"
CCronLogger.errorToConsole(errMsg)
CCronLogger.error(errMsg)
lineNr += 1
from CronLogger import CCronLogger
try:
import dominate
from dominate.tags import *
except ImportError:
CCronLogger.errorToConsole("You don't have required Python module.")
CCronLogger.debugToConsole("Required modules are: dominate")
class CCronReport:
def __init__(self,options, newCrontab, statsBefore, statsAfter, shiftVector, cronList):
self.setOptions(options)
self.newCrontab=newCrontab
self.statsBefore=statsBefore
self.statsAfter=statsAfter
self.shiftVector=shiftVector
self.cronList=cronList
def setOptions(self,options):
try:
self._outputDir=options["outputdir"]
CCronLogger.debug("Output directory: "+self._outputDir)
except KeyError:
CCronLogger.info("Output directory directive not found. Setting to default: working directory.")
self._outputDir="./"
try:
self._graphName=options["graphname"]
CCronLogger.debug("Graph name: "+self._graphName)
except KeyError:
CCronLogger.info("Graph name directive not found. Setting to default: graph.pdf.")
def plotCronJobs(self):
if len(self._cronList) == 0:
CCronLogger.errorToConsole("No viable cron jobs to plot and analyze.")
return False,None,None,None
height_graph=2+ (len(self._cronList) / 2)
#inicializace
fig = plt.figure(figsize=(8,height_graph))
ax = fig.add_subplot(111)
self._countTotalOverlaps()
self._setLineWidth()
firstTime=datetime.datetime.now().replace(year=9999)
lastTime=datetime.datetime.now().replace(year=1)
cronNameList=[]
yLabels=[]
cronCounter=0
cronID=1
counter=0
shown=False
print("Plotting cron jobs.")
for cron in self._cronList:
if self._ignoreShort and cron.duration < timedelta(seconds=self._cronLenLimit):
CCronLogger.debug("Cron job "+str(cron)+" is too short, it will be skipped.")
continue
#pokud je prikaz moc dlouhy, orizni ho
if len(cron.command) > 30:
cronNameList.append(cron.command[0:27]+" ...")
else:
cronNameList.append(cron.command)
if cron.overlaps > 0:
CCronLogger.info("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
CCronLogger.infoToConsole("Cron job "+str(cron)+" starts before previous run isn't finished. It should be manually optimized.")
yLabels.append(cronID + (cron.overlaps / 2) )
jobID=0
for job in cron.cronJobList:
self._weights.append([job.startTime,job.endTime,cron.weight])
self._endPoints[job.startTime]=None
self._endPoints[job.endTime]=None
if job.startTime<firstTime:
firstTime=job.startTime
if job.endTime>lastTime:
lastTime=job.endTime
#cron joby se prekryvaji
if cron.overlaps>0:
#vykresli je nad sebe, cervene
self._drawLine(cronID + (jobID % (cron.overlaps + 1) ),job.startTime,job.endTime,"r")
#crony se neprekryvaji
else:
self._drawLine(cronID,job.startTime,job.endTime,"k")
#prubeh
progress=int(counter*100/self.cronJobsTotal)
if not progress%5:
if not shown:
CCronLogger.debugToConsoleWithLineFeed("Progress: "+str(progress)+"%")
shown=True
else:
shown=False
counter+=1
jobID+=1
cronID+=(cron.overlaps+1)
cronCounter+=1
#pokud to neni posledni vykreslovany cron, udelej vodorovnou caru
if cronCounter!=len(self._cronList):
plt.axhline(cronID - 0.5,color="black")
CCronLogger.debugToConsole("Progress: 100%")
if counter == 0:
CCronLogger.error("No crons to plot.")
CCronLogger.errorToConsole("No crons to plot.")
return False,None,None,None
#osaX = cas
ax.xaxis_date()
#format casu na ose
length=lastTime-firstTime
self.startTime=firstTime
self.endTime=lastTime
self.duration=length
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
delta=(lastTime-firstTime)/100
#rozsah osy y
plt.ylim(-0.5,cronID)
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
plt.xlim(firstTime-delta, lastTime+delta)
#popis osyX
plt.xlabel('Time')
#plt.ylabel("Cron jobs")
fig.autofmt_xdate()
centers, weights, maxWeight, minWeight, summedWeights,weightsForStatistics = self.makeWeightIntervals()
std=self.createStatistics(maxWeight, weightsForStatistics)
loadPeaks = self.findLoadPeaks(summedWeights)
if len(loadPeaks) > 0:
yLabels.insert(0, 0)
cronNameList.insert(0, "Bottlenecks")
plt.axhline(0.5,color="black",lw=2.5)
loadPeaksLen=timedelta(seconds=0)
totalWeight=0
for peak in loadPeaks:
plt.hlines(0, peak[0], peak[1], "r", lw=self.lineWidth)
startTime=peak[0].strftime("%d.%m.%Y %H:%M:%S")
endTime=peak[1].strftime("%d.%m.%Y %H:%M:%S")
totalWeight+= peak[2]
loadPeaksLen+=(peak[1]-peak[0])
CCronLogger.info("Found bottleneck: "+startTime+" <---> "+endTime)
CCronLogger.info("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
CCronLogger.infoToConsole("Total length of load peaks: "+str(loadPeaksLen)+" of total weight: "+str(totalWeight))
#popisky osyY: 1.arg list ID (unikatni cronjoby), 2.arg = list popisku (nazvy cronjobu)
plt.yticks(yLabels, cronNameList)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+"improved_"+self._graphName)
try:
savefig(self._outputDir+"improved_"+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
CCronLogger.infoToConsole("Saving plot with cron jobs to: "+self._outputDir+self._graphName)
try:
savefig(self._outputDir+self._graphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()
self.plotWeightGraph(length, centers, weights, maxWeight)
return True,std,maxWeight,minWeight
def plotWeightGraph(self, length,centers, weights, maxWeight):
#inicializace
width = int(self._duration*10)
if width > 25:
width=25
fig = plt.figure(figsize=(width ,6))
ax = fig.add_subplot(111)
thresholds=asarray([self._threshold]*len(weights))
plt.plot(centers,weights,color="k",lw=self.weightGraphLineWidth)
plt.fill_between(centers,weights,thresholds,where=weights>thresholds,interpolate=True,color="r")
ax.xaxis_date()
dateFormat = DateFormatter(self._timeFormat)
#nastaveni osyX format casu
ax.xaxis.set_major_formatter(dateFormat)
#rezerva na oseX
#rozsah osyX (zacatecni,konecny cas) + nejaka rezerva
#popis osyX
plt.xlabel('Time')
plt.ylabel("Load")
fig.autofmt_xdate()
padd=maxWeight/10
plt.ylim(0,maxWeight+padd)
#autoupraveni odsazeni kolem grafu, aby se tam veslo vsechno
plt.tight_layout()
#export do pdf
if self.plotBetter:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+"improved_"+self._weightGraphName)
try:
savefig(self._outputDir+"improved_"+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
else:
CCronLogger.info("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
CCronLogger.infoToConsole("Saving plot with weights to: "+self._outputDir+self._weightGraphName)
try:
savefig(self._outputDir+self._weightGraphName)
except:
CCronLogger.errorToConsole("Error while saving image.")
CCronLogger.error("Error while saving image.")
#ukazani grafu v Tk
if self._showImages:
plt.show()