def generate(self):
if self.filenamein != None and self.filenameout != None:
# Read all the information
jobs = {}
attempts = []
nodes = {}
power = []
maxpower = 0.0
for line in self.filein.readlines():
if line.startswith('Job'):
ret = self.getDict(line.split(' ')[1:-1])
jobId = ret['JOBID']
if jobId not in jobs:
jobs[jobId] = Job(jobId=jobId)
jobs[jobId].status = ret['JOB_STATUS']
jobs[jobId].submit = int(ret['SUBMIT_TIME'])
jobs[jobId].start = int(ret['START_TIME'])
jobs[jobId].finish = int(ret['FINISH_TIME'])
'''
job.nmapsapprox = 0
job.nredsapprox = 0
for attempt in attempts:
if job.jobId == attempt.getJobId():
if attempt.isMap():
job.nmaps += 1
if attempt.approx or attempt.status == Job.Status.DROPPED:
job.nmapsapprox += 1
if attempt.isMap():
job.nreds += 1
if attempt.approx or attempt.status == Job.Status.DROPPED:
job.nredsapprox += 1
job.quality = 1.0 - (1.0*job.nmapsapprox/job.nmaps)
'''
elif line.startswith('Task'):
pass
elif line.startswith('MapAttempt') or line.startswith('ReduceAttempt'):
ret = self.getDict(line.split(' ')[1:-1])
attempt = Attempt()
attempt.attemptId = ret['TASK_ATTEMPT_ID']
attempt.start = int(ret['START_TIME'])
attempt.finish = int(ret['FINISH_TIME'])
attempt.approx = (ret['APPROXIMATED'] == 'True' or ret['APPROXIMATED'] == 'true')
attempt.status = ret['TASK_STATUS']
attempt.nodeId = ret['HOSTNAME']
attempts.append(attempt)
# Update job information
jobId = attempt.getJobId()
if jobId not in jobs:
jobs[jobId] = Job(jobId=jobId)
jobs[jobId].addAttempt(attempt)
elif line.startswith('Node'):
ret = self.getDict(line.split(' ')[1:-1])
nodeId = ret['HOSTNAME']
if nodeId not in nodes:
nodes[nodeId] = []
nodes[nodeId].append((int(ret['TIME']), ret['STATUS']))
elif line.startswith('Power'):
ret = self.getDict(line.split(' ')[1:-1])
power.append((int(ret['TIME']), float(ret['POWER'])))
maxpower = max(maxpower, float(ret['POWER']))
# Node -> Attempts
nodeAttempts = {}
totalAttemptsApprox = 0
totalAttemptsDropped = 0
totalAttemptsPrecise = 0
for attempt in attempts:
if attempt.approx:
totalAttemptsApprox += 1
if attempt.status == Job.Status.DROPPED:
totalAttemptsDropped += 1
if attempt.status != Job.Status.DROPPED and not attempt.approx:
totalAttemptsPrecise += 1
if attempt.nodeId not in nodeAttempts:
nodeAttempts[attempt.nodeId] = []
nodeAttempts[attempt.nodeId].append(attempt)
if 'None' in nodeAttempts:
del nodeAttempts['None']
totalJobTime = 0
totalJobRunTime = 0
totalQuality = 0.0
for jobId in jobs:
job = jobs[jobId]
totalJobTime += job.finish - job.submit
totalJobRunTime += job.finish - job.start
totalQuality += job.getQuality()
totalJobTime = totalJobTime/len(jobs)
totalJobRunTime = totalJobRunTime/len(jobs)
totalJobQuality = totalQuality/len(jobs)
# Power
power = sorted(power, key=itemgetter(0))
totalenergy = 0.0
for t, p in power:
totalenergy += p
totaltime = power[-1][0]
# Generate output HTML
self.fileout.write('<html>\n')
self.fileout.write('<head>\n')
self.fileout.write('<link rel="stylesheet" type="text/css" href="http://sol018:50030/static/hadoop.css">\n')
self.fileout.write('<title>Execution profile</title>\n')
self.fileout.write('</head>\n')
self.fileout.write('<body>\n')
# Summary
self.fileout.write('<h1>Summary</h1>\n')
self.fileout.write('<ul>\n')
self.fileout.write(' <li>Jobs: %d</li>\n' % len(jobs))
self.fileout.write(' <ul>\n')
self.fileout.write(' <li>Average turn-around time: %.1fs</li>\n' % totalJobTime)
self.fileout.write(' <li>Average runtime: %.1fs</li>\n' % totalJobRunTime)
self.fileout.write(' <li>Average quality: %.1f%%</li>\n' % totalJobQuality)
self.fileout.write(' </ul>\n')
self.fileout.write(' <li>Attempts: %d</li>\n' % len(attempts))
self.fileout.write(' <ul>\n')
self.fileout.write(' <li>Precise: %d</li>\n' % totalAttemptsPrecise)
self.fileout.write(' <li>Approximations: %d</li>\n' % totalAttemptsApprox)
self.fileout.write(' <li>Dropped: %d</li>\n' % totalAttemptsDropped)
self.fileout.write(' </ul>\n')
self.fileout.write(' <li>Energy: %.1fWh</li>\n' % (totalenergy/3600.0))
self.fileout.write(' <li>Efficiency (energy/quality): %.1f</li>\n' % ((totalenergy/3600.0)/totalJobQuality))
self.fileout.write(' <li>Efficiency (energy<sup>2</sup>/quality): %.1f</li>\n' % (pow(totalenergy/3600.0, 2)/totalJobQuality))
self.fileout.write(' <li>Efficiency (energy/quality<sup>2</sup>): %.1f</li>\n' % (totalenergy/3600.0/pow(totalJobQuality, 2)))
self.fileout.write(' <li>Time: %s</li>\n' % timeStr(totaltime))
self.fileout.write('</ul>\n')
# Jobs
if self.plotJobs:
self.fileout.write('<h1>Jobs</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
self.fileout.write('<tr><th width="100px">Id</th><th width="100px">Quality</th><th width="100px">Time</th></tr>\n')
for job in sorted(jobs.values(), key=attrgetter('finish')):
self.fileout.write('<tr><td>%s</td><td align="right">%.1f%%</td><td align="right">%d+%ds </td><td>%s</td></tr>\n' %(job.jobId, job.getQuality(), job.start-job.submit, job.finish-job.start, self.getJobGraph(job)))
self.fileout.write('</table>\n')
# Tasks
if self.plotTasks:
self.fileout.write('<h1>Tasks</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
for attempt in sorted(attempts, key=attrgetter('attemptId')):
self.fileout.write('<tr><td>'+self.getTaskGraph(attempt)+'</td></tr>\n')
self.fileout.write('</table>\n')
# Nodes -> Tasks
if self.plotNodeTasks:
self.fileout.write('<h1>Node → Tasks</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
for nodeId in sorted(nodeAttempts):
self.fileout.write('<tr><td valign="top">'+nodeId+'</td>')
self.fileout.write('<td>')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
for attempt in sorted(nodeAttempts[nodeId], key=attrgetter('start', 'attemptId')):
self.fileout.write('<tr><td>'+self.getTaskGraph(attempt)+'</td></tr>\n')
self.fileout.write('</table>\n')
self.fileout.write('</td>')
self.fileout.write('</tr>')
self.fileout.write('</table>\n')
# Nodes
if self.plotNodes:
self.fileout.write('<h1>Nodes</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
for nodeId in sorted(nodeAttempts):
self.fileout.write('<tr><td valign="top">'+nodeId+'</td>')
self.fileout.write('<td>')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
# Create slots
slots = []
for attempt in nodeAttempts[nodeId]:
added = False
for slot in slots:
if slot[-1].finish <= attempt.start:
slot.append(attempt)
added = True
break
if not added:
# Every slot is full, create a new one
slots.append([attempt])
# Draw slots
for slot in slots:
self.fileout.write('<tr><td>'+self.getTaskGraphList(slot)+'</td></tr>')
self.fileout.write('</table>\n')
self.fileout.write('</td>')
self.fileout.write('</tr>')
self.fileout.write('</table>\n')
# Nodes ON/OFF
if self.plotNodesStatus:
self.fileout.write('<h1>Nodes status</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
for nodeId in sorted(nodes):
self.fileout.write('<tr><td valign="top">'+nodeId+'</td>') # nodes[nodeId]
self.fileout.write('<td>'+self.getNodeStatusGraphList(nodes[nodeId])+'</td></tr>')
self.fileout.write('</table>\n')
# Power
if self.plotPower:
# Compress data to fit into the screen
auxpower = []
granularity=int(ceil(power[-1][0] / 1000.0))
for i in range(0, len(power)/granularity):
avgpower = []
for j in range(i*granularity, (i+1)*granularity):
avgpower.append(power[j][1])
avgpower = 1.0*sum(avgpower)/len(avgpower)
auxpower.append((power[i*granularity][0], avgpower))
power = auxpower
# Draw
self.fileout.write('<h1>Power</h1>\n')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">\n')
self.fileout.write('<tr>')
for t, p in power:
self.fileout.write('<td width="1px">')
self.fileout.write('<table border="0" cellspacing="0" cellpadding="0">')
self.fileout.write('<tr height="%dpx"><td width="1px"/></tr>' % (100.0*(maxpower-p)/maxpower))
self.fileout.write('<tr height="%dpx"><td width="1px" style="background-color:#0000FF"/>' % (100.0*p/maxpower))
self.fileout.write('</tr></table>')
self.fileout.write('</td>\n')
self.fileout.write('<tr>')
self.fileout.write('</table>\n')
self.fileout.write('</body>\n')
self.fileout.write('</html>\n')
