def getTaskManager(dbconn=None):
''' Open task manager '''
dbconn = dbconn or options.dbconn
try:
return TaskManager(dbconn)
except:
print('ERROR: Unable to access task manager database %s' % dbconn)
sys.exit(1)
def default(self,run):
"""Return a summary gif image for the run specified in the URL. If no such image exists (or the run
doesn't exist), currently returns a zero-length image."""
cherrypy.response.headers['Content-Type'] = "image/gif"
try:
with TaskManager(self.globalConfig['taskDb']) as taskman:
t = taskman.taskIterDict('*',['where RUNNR =',DbParam(int(run)),"and TASKNAME like 'DB_BEAMSPOT%%' order by UPDATED desc"]).next()
wantedFile = 'PlotBeamSpot.gif'
for f in t['RESULTFILES'].split():
if f[-len(wantedFile):] == wantedFile: break
path = '/'.join([self.globalConfig['jobDir'],t['DSNAME'],t['TASKNAME'],f])
return open(path,'r').read()
except:
return None
def content(self, **args):
with TaskManager(self.globalConfig['taskDb']) as taskman:
if 'r' in args:
table = taskCountForRun % (taskman.getNTasks(
['where RUNNR =', DbParam(args['r'])]), args['r'])
taskIter = taskman.taskIterDict(
'*',
['where RUNNR =',
DbParam(args['r']), 'order by UPDATED'])
elif 'd' in args:
table = taskCountForDS % (taskman.getNTasks(
["where DSNAME like '%%%s%%'" % args['d']]), args['d'])
taskIter = taskman.taskIterDict('*', [
"where DSNAME like '%%%s%%' order by UPDATED" % args['d']
])
else:
limit = int(args['limit']) if 'limit' in args else 50
if not limit:
limit = 99999999
table = taskCount % taskman.getNTasks()
taskIter = taskman.taskIterDict(
qual=('order by UPDATED desc', ))
else:
table = taskCountLimit % (taskman.getNTasks(), limit)
taskIter = taskman.taskIterDict(
qual=('order by UPDATED desc', ), limit=limit)
table += tableHeader
for t in taskIter:
dsname = t['DSNAME']
taskname = t['TASKNAME']
table += "<tr>"
table += "<td>%s</td>" % (time.ctime(t['UPDATED']))
table += "<td>%s</td>" % (dsname)
table += "<td>%s</td>" % (taskname)
table += "<td>%s</td>" % ('<br>'.join(t['ATLREL'].split(';')))
table += "<td>%s</td>" % (t['NJOBS'])
stat = t['STATUS']
table += '<td class="%s">%s</td>' % (
getStatusClass(stat), getKey(TaskManager.StatusCodes, stat))
table += '<td>'
table += ' <a href="../details?d=%s&t=%s">Details</a>' % (dsname,
taskname)
table += ' <a href="../files?u=%s/%s">Files</a>' % (dsname,
taskname)
table += '</td>'
table += "<td>%s</td>" % (blankIfNone(t['RESULTLINKS']))
table += "</tr>\n"
table += "</tbody></table>\n"
return table
# Temporary warning. TODO: still needed?
if grid_mode and opts.autoconfparams:
print "WARNING: Automatic configuration of parameters such as DetDescrVersion doesn't work yet on the grid!"
print " Please be sure the values of each of the following parameters are specified explicitly above,"
print " unless the defaults in the job option template are correct:\n"
print " ", opts.autoconfparams
print
print len(files), "input file(s)/dataset found."
print
if not opts.testonly:
runner.configure()
if opts.taskdb != 'None':
try:
with TaskManager.TaskManager(opts.taskdb) as taskman:
taskman.addTask(dsname,
taskname,
joboptiontemplate,
runner.getParam('release'),
runner.getNJobs(),
opts.postprocsteps,
comment=cmd)
except:
print 'WARNING: Unable to add task to task manager database ' + opts.taskdb
runner.run()
if opts.legacy_dowait and not grid_mode:
if not opts.legacy_interactive: runner.wait()
print
print "Job directories in %s for this task:" % workdir
print
def content(self,**args):
if 'type' in args:
type = args['type']
else:
type = ''
limit = int(args['limit']) if 'limit' in args else 50
with TaskManager(self.globalConfig['taskDb']) as taskman:
if not limit:
limit = 99999999
table = runCount % (taskman.getCount('distinct(runnr)',[ "where TASKNAME like '%s%%'" % type]), type)
else:
table = runCountLimit % (taskman.getCount('distinct(runnr)',[ "where TASKNAME like '%s%%'" % type]), type,limit)
table += tableHeader % type
for r in taskman.taskIterDict('distinct(DSNAME),RUNNR',["where TASKNAME like '%s%%' order by RUNNR desc" % type], limit):
runnr = r['RUNNR']
dsname = r['DSNAME']
try:
stream = dsname.split('.')[-1].split('_')[-1]
except:
stream = ''
if not runnr: continue
table += "<tr>"
table += '<td><a href="http://atlas-runquery.cern.ch/query.py?q=find+r+%s+/+sh+lhc+all+and+r+and+t+and+mag+and+dq+idbs,pix,sct">%s</a></td>' % (runnr,runnr)
table += '<td>%s</td>' % stream
try:
t = taskman.taskIterDict('*',['where RUNNR =',DbParam(runnr),'and DSNAME =',DbParam(dsname),"and TASKNAME like '%s%%' order by UPDATED desc" % type]).next()
taskName = t['TASKNAME']
datatag = taskName.split('.')[-1].split('_')[0]
if taskName[:11] == 'DB_BEAMSPOT.':
# Special naming convention for T0 beam spot jobs
monTaskName = 'MON.%s.%s' % (taskName,datatag)
elif taskName[:-1] == 'REPROHIMAR2011_BEAMSPOT.r2074.v':
# Kludge to fix inconsistent naming for Mar 2011 HI reprocessing jobs
monTaskName = 'MON.REPROHIMAR2011_BEAMSPOT.r2074'
else:
# Other monitoring jobs
monTaskName = 'MON.%s%%' % (taskName)
try:
m = taskman.taskIterDict('*',['where RUNNR =',DbParam(runnr),'and DSNAME =',DbParam(dsname),'and TASKNAME like ',DbParam(monTaskName),'order by UPDATED desc']).next()
stat = m['STATUS']
monStatus = '<td class="%s"><a href="../details?d=%s&t=%s">%s</a></td>' % (getStatusClass(stat),t['DSNAME'],m['TASKNAME'],getKey(TaskManager.StatusCodes,stat))
monResults = "<td>%s</td>" % (blankIfNone(m['RESULTLINKS']))
except:
monStatus = '<td></td>'
monResults = '<td></td>'
table += '<td><a href="../details?d=%s&t=%s">%s</a></td>' % (t['DSNAME'],taskName,taskName)
table += "<td>%s</td>" % (time.ctime(t['UPDATED']))
stat = t['STATUS']
table += '<td class="%s">%s</td>' % (getStatusClass(stat),getKey(TaskManager.StatusCodes,stat))
table += "<td>%s</td>" % (blankIfNone(t['RESULTLINKS']))
table += "<td>"
cooltags = t['COOLTAGS']
if not cooltags:
cooltags = '' # make sure it is not None
for tag in cooltags.split():
table += '<a href="http://atlas-runquery.cern.ch/query.py?q=find+run+%s+/+show+bs+%s">%s</a> ' % (runnr,tag,tag)
table += '<br>'
table += "</td>"
except Exception as e:
table += "<td>%s</td>" % str(e)
table += "<td></td>"
table += "<td></td>"
table += "<td></td>"
table += "<td></td>"
monStatus = '<td></td>'
monResults = '<td></td>'
table += monStatus
table += monResults
table += '<td><a href="../jobs?r=%s">all tasks</a></td>' % runnr
table += "</tr>\n"
table += "</tbody></table>\n"
return table
def content(self, **args):
if not 'd' in args:
return error % ('CONFIGURATION ERROR: No data set name specified')
if not 't' in args:
return error % ('CONFIGURATION ERROR: No task name specified')
with TaskManager(self.globalConfig['taskDb']) as taskman:
t = taskman.getTaskDict(args['d'], args['t'])
if not t:
return error % ('ERROR: Task %s/%s not found' %
(args['d'], args['t']))
p = '<div class="text"><h3>Summary for task %s/%s (task id %s):</h3></div>\n' % (
args['d'], args['t'], t['TASKID'])
p += '<table>\n'
p += tableRow % ('Dataset name', t['DSNAME'])
p += tableRow % ('Run number', blankIfNone(t['RUNNR']))
p += tableRow % ('Task name', t['TASKNAME'])
stat = t['STATUS']
p += tableRowStatus % ('Task status', getStatusClass(stat),
getKey(TaskManager.StatusCodes, stat))
p += tableRow % ('On-disk status',
getKey(TaskManager.OnDiskCodes, t['ONDISK']))
p += tableRow % ('Template', t['TEMPLATE'])
p += tableRow % ('Postprocessing steps', t['TASKPOSTPROCSTEPS'])
p += tableRow % ('ATLAS release', t['ATLREL'])
p += tableRow % ('# result files', t['NRESULTFILES'])
p += tableRow % ('Result files',
fileListSnippet(t['RESULTFILES'], t['DSNAME'],
t['TASKNAME'],
self.globalConfig['jobDir']))
p += tableRow % ('Result links', t['RESULTLINKS'])
cooltags = t['COOLTAGS'] or ''
p += tableRow % ('COOL tags', '<br>'.join(cooltags.split()))
p += tableRow % ('# jobs', t['NJOBS'])
p += tableRow % ('# jobs submitted', t['NJOBS_SUBMITTED'])
p += tableRow % ('# jobs running', t['NJOBS_RUNNING'])
p += tableRow % ('# jobs postprocessing', t['NJOBS_POSTPROC'])
p += tableRow % ('# jobs failed', t['NJOBS_FAILED'])
p += tableRow % ('# jobs completed', t['NJOBS_COMPLETED'])
p += tableRow % ('Comment', t['TASKCOMMENT'])
p += tableRow % (
'Created', '%s by %s on host %s' %
(time.ctime(t['CREATED']), t['CREATED_USER'], t['CREATED_HOST']))
p += tableRow % ('Last update', time.ctime(t['UPDATED']))
p += '</table>\n'
p += '<div class="text"><h3>Job details (may be more up-to-date than summary data above):</h3></div>\n'
p += tableHeader
a = JobAnalyzer(self.globalConfig['jobDir'], t['DSNAME'],
t['TASKNAME'])
for j in a.jobList():
p += "<tr>"
p += "<td>%s</td>" % (j)
(stat, exitcode) = a.jobStatus(j)
p += '<td class="%s">%s</td>' % (
getStatusClass(stat), getKey(TaskManager.StatusCodes, stat))
p += "<td>%s</td>" % (exitcode)
p += '<td><a href="../files?u=%s/%s/%s">Files</a></td>' % (
t['DSNAME'], t['TASKNAME'], j)
p += "</tr>\n"
p += "</tbody></table>\n"
return p
#
# Initialize a new task database
#
if cmd == 'init':
if cmdargs: parser.error('Command init does not take arguments')
try:
dbtype, dbname = TaskManager.parseConnectionInfo(options.dbconn)
except ValueError as e:
sys.exit('ERROR: {}'.format(e))
print('Will initialise schema for database: {}:{}'.format(dbtype, dbname))
print()
print('Checking for existing database ...')
try:
with TaskManager(options.dbconn):
pass
except ValueError as e:
print('Test connection failed: {}'.format(e))
else:
print('Test connection succeeded: the database already exists!')
if not options.batch:
a = raw_input(
'\nRECREATING TASK DATABASE SCHEMA - ANY EXISTING DATA WILL BE ERASED!\n\nARE YOU REALLY ABSOLUTELY SURE [n] ? '
)
if a != 'y': sys.exit('ERROR: Rebuilding aborted by user')
print()
with TaskManager(options.dbconn, createDatabase=True):
pass
sys.exit(0)