def harvesters(request):
import json
valid, response = initRequest(request)
#query, extra, LAST_N_HOURS_MAX = setupView(request, wildCardExt=True)
extra = '1=1'
xurl = extensibleURL(request)
if 'instance' in request.session['requestParams']:
instance = request.session['requestParams']['instance']
# if (('HTTP_ACCEPT' in request.META) and (request.META.get('HTTP_ACCEPT') in ('text/json', 'application/json'))) or ('json' in request.session['requestParams']):
# data = getCacheEntry(request, instance,isData=True)
# import json
# return HttpResponse(data, content_type='text/html')
data = getCacheEntry(request, "harvester")
if data is not None:
import json
data = json.loads(data)
data['request'] = request
response = render_to_response('harvesters.html',
data,
content_type='text/html')
patch_response_headers(
response,
cache_timeout=request.session['max_age_minutes'] * 60)
endSelfMonitor(request)
return response
if ('workersstats' in request.session['requestParams']
and 'instance' in request.session['requestParams']):
harvsterworkerstats = []
tquery = {}
tquery['harvesterid'] = instance
limit = 100
if 'limit' in request.session['requestParams']:
limit = request.session['requestParams']['limit']
harvsterworkerstat = HarvesterWorkerStats.objects.filter(
**tquery).values(
'computingsite', 'resourcetype', 'status', 'nworkers',
'lastupdate').filter(**tquery).extra(
where=[extra]).order_by('-lastupdate')[:limit]
# dialogs.extend(HarvesterDialogs.objects.filter(**tquery).values('creationtime','modulename', 'messagelevel','diagmessage').filter(**tquery).extra(where=[extra]).order_by('-creationtime'))
old_format = '%Y-%m-%d %H:%M:%S'
new_format = '%d-%m-%Y %H:%M:%S'
for stat in harvsterworkerstat:
stat['lastupdate'] = datetime.strptime(str(
stat['lastupdate']), old_format).strftime(new_format)
harvsterworkerstats.append(stat)
return HttpResponse(json.dumps(harvsterworkerstats,
cls=DateTimeEncoder),
content_type='text/html')
if ('dialogs' in request.session['requestParams']
and 'instance' in request.session['requestParams']):
dialogs = []
tquery = {}
tquery['harvesterid'] = instance
limit = 100
if 'limit' in request.session['requestParams']:
limit = request.session['requestParams']['limit']
dialogsList = HarvesterDialogs.objects.filter(**tquery).values(
'creationtime', 'modulename', 'messagelevel',
'diagmessage').filter(**tquery).extra(
where=[extra]).order_by('-creationtime')[:limit]
# dialogs.extend(HarvesterDialogs.objects.filter(**tquery).values('creationtime','modulename', 'messagelevel','diagmessage').filter(**tquery).extra(where=[extra]).order_by('-creationtime'))
old_format = '%Y-%m-%d %H:%M:%S'
new_format = '%d-%m-%Y %H:%M:%S'
for dialog in dialogsList:
dialog['creationtime'] = datetime.strptime(
str(dialog['creationtime']),
old_format).strftime(new_format)
dialogs.append(dialog)
return HttpResponse(json.dumps(dialogs, cls=DateTimeEncoder),
content_type='text/html')
if ('dt' in request.session['requestParams']
and 'tk' in request.session['requestParams']):
tk = request.session['requestParams']['tk']
data = getCacheEntry(request, tk, isData=True)
return HttpResponse(data, content_type='text/html')
lastupdateCache = ''
workersListCache = []
data = {}
setCacheEntry(request,
instance,
json.dumps(data, cls=DateEncoder),
1,
isData=True)
workersListisEmty = True
if 'status' not in request.session[
'requestParams'] and 'computingsite' not in request.session[
'requestParams'] and 'days' not in request.session[
'requestParams']:
data = getCacheEntry(request, instance, isData=True)
if data is not None and data != "null":
if 'lastupdate' in data:
data = json.loads(data)
lastupdateCache = data['lastupdate'].replace('T', ' ')
lastupdateCache = """ AND "wrklastupdate" >= to_date('%s','yyyy-mm-dd hh24:mi:ss')""" % (
lastupdateCache)
workersListCache = data['workersList']
workersListisEmty = False
tmpworkerList = data['workersList'].keys()
for worker in tmpworkerList:
if datetime.strptime(
data['workersList'][worker]['wrklastupdate'],
'%d-%m-%Y %H:%M:%S'
) < datetime.now() - timedelta(days=60):
del data['workersList'][worker]
else:
lastupdateCache = ''
workersListCache = []
status = ''
computingsite = ''
workerid = ''
days = ''
if 'status' in request.session['requestParams']:
status = """AND status like '%s'""" % (str(
request.session['requestParams']['status']))
if 'computingsite' in request.session['requestParams']:
computingsite = """AND computingsite like '%s'""" % (str(
request.session['requestParams']['computingsite']))
if 'workerid' in request.session['requestParams']:
workerid = """AND workerid in (%s)""" % (
request.session['requestParams']['workerid'])
if 'days' in request.session['requestParams']:
days = """AND to_date("wrklastupdate", 'dd-mm-yyyy hh24:mi:ss') > sysdate - %s """ % (
request.session['requestParams']['days'])
sqlquery = """
select * from (SELECT
ff.harvester_id,
ff.description,
to_char(ff.starttime, 'dd-mm-yyyy hh24:mi:ss') as "insstarttime",
ff.owner,
ff.hostname,
ff.sw_version,
ff.commit_stamp,
gg.workerid,
to_char((select max(lastupdate) from atlas_panda.harvester_workers where harvesterid like '%s'), 'dd-mm-yyyy hh24:mi:ss') as "inslastupdate",
gg.status,
gg.batchid,
gg.nodeid,
gg.queuename,
gg.computingsite,
to_char(gg.submittime, 'dd-mm-yyyy hh24:mi:ss') as "submittime",
to_char(gg.lastupdate , 'dd-mm-yyyy hh24:mi:ss') as "wrklastupdate",
to_char(gg.starttime , 'dd-mm-yyyy hh24:mi:ss') as "wrkstarttime",
to_char(gg.endtime, 'dd-mm-yyyy hh24:mi:ss') as "wrkendtime",
gg.ncore,
gg.errorcode,
gg.stdout,
gg.stderr,
gg.batchlog,
gg.resourcetype,
gg.nativeexitcode,
gg.nativestatus,
gg.diagmessage,
gg.computingelement,
gg.njobs
FROM
atlas_panda.harvester_workers gg,
atlas_panda.harvester_instances ff
WHERE
ff.harvester_id = gg.harvesterid) where harvester_id like '%s' %s %s %s %s %s
order by workerid DESC
""" % (str(instance), str(instance), status, computingsite, workerid,
lastupdateCache, days)
workersList = []
cur = connection.cursor()
cur.execute(sqlquery)
columns = [str(i[0]).lower() for i in cur.description]
workersDictinoary = {}
timeLastUpdate = ''
if workersListisEmty == False:
for worker in cur:
object = {}
object = dict(zip(columns, worker))
workersListCache[int(object['workerid'])] = object
timeLastUpdate = object['inslastupdate']
workersList = workersListCache.values()
workersDictinoary = workersListCache
else:
for worker in cur:
object = {}
object = dict(zip(columns, worker))
workersDictinoary[int(object['workerid'])] = object
workersList.append(object)
timeLastUpdate = object['inslastupdate']
# dbCache = {
# "workersList": workersDictinoary,
# "lastupdate": timeLastUpdate
# }
# print len(workersListCache)
# if 'status' not in request.session['requestParams'] and 'computingsite' not in request.session['requestParams'] and 'workerid' not in request.session['requestParams'] :
# setCacheEntry(request, instance, json.dumps(dbCache, cls=DateEncoder), 86400, isData=True)
statuses = {}
computingsites = {}
workerIDs = set()
generalInstanseInfo = {}
if 'display_limit_workers' in request.session['requestParams']:
display_limit_workers = int(
request.session['requestParams']['display_limit_workers'])
else:
display_limit_workers = 30000
generalWorkersFields = [
'workerid', 'status', 'batchid', 'nodeid', 'queuename',
'computingsite', 'submittime', 'wrklastupdate', 'wrkstarttime',
'wrkendtime', 'ncore', 'errorcode', 'stdout', 'stderr', 'batchlog',
'resourcetype', 'nativeexitcode', 'nativestatus', 'diagmessage',
'njobs', 'computingelement'
]
generalWorkersList = []
wrkPandaIDs = {}
for i, worker in enumerate(workersList):
object = {}
computingsites.setdefault(worker['computingsite'],
[]).append(worker['workerid'])
statuses.setdefault(worker['status'],
[]).append(worker['workerid'])
# if worker['njobs'] is not None:
# wrkPandaIDs[worker['workerid']] = worker['njobs']
# else: wrkPandaIDs[worker['workerid']] = 0
#workerIDs.add(worker['workerid'])
for field in generalWorkersFields:
if worker[field] is not None:
object[field] = worker[field]
else:
object[field] = 0
generalWorkersList.append(object)
if i == len(workersList) - 1:
for computingsite in computingsites.keys():
computingsites[computingsite] = len(
computingsites[computingsite])
for status in statuses.keys():
statuses[status] = len(statuses[status])
generalInstanseInfo = {
'HarvesterID': worker['harvester_id'],
'Description': worker['description'],
'Starttime': worker['insstarttime'],
'Owner': worker['owner'],
'Hostname': worker['hostname'],
'Lastupdate': worker['inslastupdate'],
'Computingsites': computingsites,
'Statuses': statuses,
'Software version': worker['sw_version'],
'Commit stamp': worker['commit_stamp']
}
# 'wrkpandaids': OrderedDict(sorted(wrkPandaIDs.items(), key=lambda x: x[1], reverse=True)[:200])
transactionKey = random.randrange(1000000)
data = {
'generalInstanseInfo': generalInstanseInfo,
'type': 'workers',
'instance': instance,
'xurl': xurl,
'tk': transactionKey,
'request': request,
'requestParams': request.session['requestParams'],
'viewParams': request.session['viewParams'],
'built': datetime.now().strftime("%H:%M:%S"),
}
setCacheEntry(request,
transactionKey,
json.dumps(generalWorkersList[:display_limit_workers],
cls=DateEncoder),
60 * 60,
isData=True)
setCacheEntry(request, 'harvester', json.dumps(data, cls=DateEncoder),
60 * 60)
endSelfMonitor(request)
return render_to_response('harvesters.html',
data,
content_type='text/html')
# elif 'instance' in request.session['requestParams'] and 'workerid' in 'instance' in request.session['requestParams']:
# pass
else:
sqlquery = """
select
R.harvid,
count(R.workid) as total,
(select cnt from (select harvid, count(*) as cnt from (
SELECT
a.harvester_id as harvid,
b.workerid as workid,
to_char(b.lastupdate, 'dd-mm-yyyy hh24:mi:ss') as alldate,
(SELECT
to_char(max(O.lastupdate), 'dd-mm-yyyy hh24:mi:ss')
FROM atlas_panda.harvester_workers O WHERE O.harvesterid = a.harvester_id Group by O.harvesterid) as recently,
a.DESCRIPTION as description
FROM
atlas_panda.harvester_workers b,
atlas_panda.harvester_instances a
WHERE a.harvester_id = b.harvesterid
) WHERE alldate = recently Group by harvid) W WHERE W.harvid=R.harvid) as recent,
R.recently,
R.sw_version,
R.commit_stamp,
R.lastupdate,
R.description
FROM (SELECT
a.harvester_id as harvid,
b.workerid as workid,
to_char(b.lastupdate, 'dd-mm-yyyy hh24:mi:ss') as alldate,
(SELECT
to_char(max(O.lastupdate), 'dd-mm-yyyy hh24:mi:ss')
FROM atlas_panda.harvester_rel_jobs_workers O where O.harvesterid = a.harvester_id Group by O.harvesterid) as recently,
a.sw_version,
a.commit_stamp,
to_char(a.lastupdate, 'dd-mm-yyyy hh24:mi:ss') as lastupdate,
a.DESCRIPTION as description
FROM
atlas_panda.harvester_workers b,
atlas_panda.harvester_instances a
WHERE a.harvester_id = b.harvesterid) R group by harvid,recently,sw_version,commit_stamp,lastupdate,description
"""
instanceDictionary = []
cur = connection.cursor()
cur.execute(sqlquery)
for instance in cur:
instanceDictionary.append({
'instance': instance[0],
'total': instance[1],
'recently': instance[2],
'when': instance[3],
'sw_version': instance[4],
'commit_stamp': instance[5],
'lastupdate': instance[6],
'descr': instance[7]
})
data = {
'instances': instanceDictionary,
'type': 'instances',
'xurl': xurl,
'request': request,
'requestParams': request.session['requestParams'],
'viewParams': request.session['viewParams']
}
#data =json.dumps(data,cls=DateEncoder)
response = render_to_response('harvesters.html',
data,
content_type='text/html')
return response
def globalshares(request):
valid, response = initRequest(request)
data = getCacheEntry(request, "globalshares")
if data is not None:
data = json.loads(data)
data['request'] = request
gsPlotData = {}
oldGsPlotData = data['gsPlotData']
for shareName, shareValue in oldGsPlotData.iteritems():
gsPlotData[str(shareName)] = int(shareValue)
data['gsPlotData'] = gsPlotData
if not valid: return response
setupView(request, hours=180 * 24, limit=9999999)
gs, tablerows = __get_hs_leave_distribution()
gsPlotData = {
} #{'Upgrade':130049 , 'Reprocessing default':568841, 'Data Derivations': 202962, 'Event Index': 143 }
for shareName, shareValue in gs.iteritems():
shareValue['delta'] = shareValue['executing'] - shareValue['pledged']
shareValue[
'used'] = shareValue['ratio'] if 'ratio' in shareValue else None
gsPlotData[str(shareName)] = int(shareValue['executing'])
for shareValue in tablerows:
shareValue['used'] = shareValue['ratio'] * Decimal(
shareValue['value']) / 100 if 'ratio' in shareValue else None
ordtablerows = {}
ordtablerows['childlist'] = []
level1 = ''
level2 = ''
level3 = ''
for shareValue in tablerows:
if len(shareValue['level1']) != 0:
level1 = shareValue['level1']
ordtablerows[level1] = {}
ordtablerows['childlist'].append(level1)
ordtablerows[level1]['childlist'] = []
if len(shareValue['level2']) != 0:
level2 = shareValue['level2']
ordtablerows[level1][level2] = {}
ordtablerows[level1]['childlist'].append(level2)
ordtablerows[level1][level2]['childlist'] = []
if len(shareValue['level3']) != 0:
level3 = shareValue['level3']
ordtablerows[level1][level2][level3] = {}
ordtablerows[level1][level2]['childlist'].append(level3)
resources_list, resources_dict = get_resources_gshare()
newTablesRow = []
for ordValueLevel1 in sorted(ordtablerows['childlist']):
for shareValue in tablerows:
if ordValueLevel1 in shareValue['level1']:
ord1Short = re.sub('\[(.*)\]', '',
ordValueLevel1).rstrip().lower()
shareValue['level'] = 'level1'
shareValue['gshare'] = ord1Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if len(ordtablerows[ordValueLevel1]['childlist']) == 0:
add_resources(ord1Short, newTablesRow, resources_list,
shareValue['level'])
else:
childsgsharelist = []
get_child_elements(ordtablerows[ordValueLevel1],
childsgsharelist)
resources_dict = get_child_sumstats(
childsgsharelist, resources_dict, ord1Short)
short_resource_list = resourcesDictToList(resources_dict)
add_resources(ord1Short, newTablesRow, short_resource_list,
shareValue['level'])
break
for ordValueLevel2 in sorted(
ordtablerows[ordValueLevel1]['childlist']):
for shareValue in tablerows:
if ordValueLevel2 in shareValue['level2']:
if len(ordtablerows[ordValueLevel1][ordValueLevel2]
['childlist']) == 0:
ord1Short = re.sub('\[(.*)\]', '',
ordValueLevel1).rstrip().lower()
ord2Short = re.sub('\[(.*)\]', '',
ordValueLevel2).rstrip().lower()
link = "?jobtype=%s&display_limit=100&gshare=%s" % (
ord1Short, ord2Short)
shareValue['link'] = link
shareValue['level'] = 'level2'
shareValue['gshare'] = ord2Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if 'level' in shareValue:
add_resources(ord2Short, newTablesRow, resources_list,
shareValue['level'])
break
for ordValueLevel3 in sorted(
ordtablerows[ordValueLevel1][ordValueLevel2]['childlist']):
for shareValue in tablerows:
if ordValueLevel3 in shareValue['level3']:
if len(ordtablerows[ordValueLevel1][ordValueLevel2]
['childlist']) > 0:
ord1Short = re.sub(
'\[(.*)\]', '',
ordValueLevel1).rstrip().lower()
ord3Short = re.sub(
'\[(.*)\]', '',
ordValueLevel3).rstrip().lower()
link = "?jobtype=%s&display_limit=100&gshare=%s" % (
ord1Short, ord3Short)
shareValue['link'] = link
shareValue['level'] = 'level3'
shareValue['gshare'] = ord3Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if 'level' in shareValue:
add_resources(ord3Short, newTablesRow,
resources_list, shareValue['level'])
break
tablerows = newTablesRow
del request.session['TFIRST']
del request.session['TLAST']
##self monitor
endSelfMonitor(request)
if (not (('HTTP_ACCEPT' in request.META) and
(request.META.get('HTTP_ACCEPT') in ('application/json')))
and ('json' not in request.session['requestParams'])):
data = {
'request': request,
'viewParams': request.session['viewParams'],
'requestParams': request.session['requestParams'],
'globalshares': gs,
'xurl': extensibleURL(request),
'gsPlotData': gsPlotData,
'tablerows': tablerows,
'built': datetime.now().strftime("%H:%M:%S"),
}
response = render_to_response('globalshares.html',
data,
content_type='text/html')
setCacheEntry(request, "globalshares", json.dumps(data,
cls=DateEncoder),
60 * 20)
patch_response_headers(
response, cache_timeout=request.session['max_age_minutes'] * 60)
return response
else:
return HttpResponse(json.dumps(gs), content_type='text/html')
def compareJobs(request):
valid, response = initRequest(request)
if not valid: return response
pandaidstr = None
if 'pandaid' in request.session['requestParams']:
pandaidstr = request.session['requestParams']['pandaid'].split('|')
else:
query = {}
query['userid'] = request.user.id
query['object'] = 'job'
try:
jobsComparison = ObjectsComparison.objects.get(**query)
pandaidstr = json.loads(jobsComparison.comparisonlist)
except ObjectsComparison.DoesNotExist:
pandaidstr = None
if not pandaidstr:
return render_to_response(
'errorPage.html',
{'errormessage': 'No pandaids for comparison provided'},
content_type='text/html')
pandaids = []
for pid in pandaidstr:
try:
pid = int(pid)
pandaids.append(pid)
except:
pass
maxNJobs = 5
if len(pandaids) > maxNJobs:
pandaids = pandaids[:maxNJobs]
jobInfoJSON = []
# Looking for a job in cache
pandaidsToBeLoad = []
for pandaid in pandaids:
data = getCacheEntry(request,
"compareJob_" + str(pandaid),
isData=True)
# data = None
if data is not None:
jobInfoJSON.append(json.loads(data))
else:
pandaidsToBeLoad.append(pandaid)
#Loading jobs info in parallel
nprocesses = maxNJobs
if len(pandaidsToBeLoad) > 0:
url_params = [('?json=1&pandaid=' + str(pid))
for pid in pandaidsToBeLoad]
pool = multiprocessing.Pool(processes=nprocesses)
jobInfoJSON.extend(pool.map(job_info_getter, url_params))
pool.close()
pool.join()
#Put loaded jobs info to cache
for job in jobInfoJSON:
setCacheEntry(request,
"compareJob_" + str(job.keys()[0]),
json.dumps(job.values()[0], cls=DateEncoder),
60 * 30,
isData=True)
compareParamNames = {
'produsername': '******',
'reqid': 'Request ID',
'jeditaskid': 'Task ID',
'jobstatus': 'Status',
'attemptnr': 'Attempt',
'creationtime': 'Created',
'waittime': 'Time to start',
'duration': 'Duration',
'modificationtime': 'Modified',
'cloud': 'Cloud',
'computingsite': 'Site',
'currentpriority': 'Priority',
'jobname': 'Name',
'processingtype': 'Type',
'transformation': 'Transformation',
'proddblock': 'Input',
'destinationdblock': 'Output',
'jobsetid': 'Jobset ID',
'batchid': 'Batch ID',
'eventservice': 'Event Service'
}
compareParams = [
'produsername', 'reqid', 'jeditaskid', 'jobstatus', 'attemptnr',
'creationtime', 'waittime', 'duration', 'modificationtime', 'cloud',
'computingsite', 'currentpriority', 'jobname', 'processingtype',
'transformation', 'proddblock', 'destinationdblock', 'jobsetid',
'batchid', 'eventservice'
]
###Excluded params because of too long values###
excludedParams = ['metadata', 'metastruct']
jobsComparisonMain = []
for param in compareParams:
row = [{'paramname': compareParamNames[param]}]
for jobd in jobInfoJSON:
job = jobd['job']
if param in job:
row.append({'value': job[param]})
else:
row.append({'value': '-'})
if len(set([d['value'] for d in row if 'value' in d])) == 1:
row[0]['mark'] = 'equal'
jobsComparisonMain.append(row)
all_params = []
for jobd in jobInfoJSON:
all_params.extend(list(jobd['job'].keys()))
all_params = sorted(set(all_params))
jobsComparisonAll = []
for param in all_params:
if param not in excludedParams:
row = [{'paramname': param}]
for jobd in jobInfoJSON:
job = jobd['job']
if param in job and job[param] is not None:
row.append({'value': job[param]})
else:
row.append({'value': '-'})
if len(set([d['value'] for d in row if 'value' in d])) == 1:
row[0]['mark'] = 'equal'
jobsComparisonAll.append(row)
xurl = extensibleURL(request)
data = {
'request': request,
'viewParams': request.session['viewParams'],
'requestParams': request.session['requestParams'],
'url': request.path,
'jobsComparisonMain': jobsComparisonMain,
'jobsComparisonAll': jobsComparisonAll,
'pandaids': pandaids,
'xurl': xurl,
'built': datetime.now().strftime("%H:%M:%S"),
}
##self monitor
endSelfMonitor(request)
response = render_to_response('compareJobs.html',
data,
content_type='text/html')
patch_response_headers(response,
cache_timeout=request.session['max_age_minutes'] *
60)
return response
def globalshares(request):
valid, response = initRequest(request)
data = getCacheEntry(request, "globalshares")
if data is not None:
data = json.loads(data)
data['request'] = request
gsPlotData = {}
oldGsPlotData = data['gsPlotData']
for shareName, shareValue in oldGsPlotData.iteritems():
gsPlotData[str(shareName)] = int(shareValue)
data['gsPlotData'] = gsPlotData
if not valid: return response
setupView(request, hours=180 * 24, limit=9999999)
gs, tablerows = __get_hs_leave_distribution()
gsPlotData = {}#{'Upgrade':130049 , 'Reprocessing default':568841, 'Data Derivations': 202962, 'Event Index': 143 }
for shareName, shareValue in gs.iteritems():
shareValue['delta'] = shareValue['executing'] - shareValue['pledged']
shareValue['used'] = shareValue['ratio'] if 'ratio' in shareValue else None
gsPlotData[str(shareName)] = int(shareValue['executing'])
for shareValue in tablerows:
shareValue['used'] = shareValue['ratio']*Decimal(shareValue['value'])/100 if 'ratio' in shareValue else None
ordtablerows ={}
ordtablerows['childlist']=[]
level1=''
level2=''
level3=''
for shareValue in tablerows:
if len(shareValue['level1'])!=0:
level1 = shareValue['level1']
ordtablerows[level1] = {}
ordtablerows['childlist'].append(level1)
ordtablerows[level1]['childlist'] = []
if len(shareValue['level2'])!=0:
level2 = shareValue['level2']
ordtablerows[level1][level2] = {}
ordtablerows[level1]['childlist'].append(level2)
ordtablerows[level1][level2]['childlist'] = []
if len(shareValue['level3'])!=0:
level3 = shareValue['level3']
ordtablerows[level1][level2][level3] = {}
ordtablerows[level1][level2]['childlist'].append(level3)
resources_list, resources_dict = get_resources_gshare()
newTablesRow =[]
for ordValueLevel1 in sorted(ordtablerows['childlist']):
for shareValue in tablerows:
if ordValueLevel1 in shareValue['level1']:
ord1Short = re.sub('\[(.*)\]', '', ordValueLevel1).rstrip().lower()
shareValue['level'] = 'level1'
shareValue['gshare'] = ord1Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if len(ordtablerows[ordValueLevel1]['childlist']) == 0:
add_resources(ord1Short,newTablesRow,resources_list,shareValue['level'])
else:
childsgsharelist = []
get_child_elements(ordtablerows[ordValueLevel1],childsgsharelist)
resources_dict = get_child_sumstats(childsgsharelist,resources_dict,ord1Short)
short_resource_list= resourcesDictToList(resources_dict)
add_resources(ord1Short, newTablesRow, short_resource_list, shareValue['level'])
break
for ordValueLevel2 in sorted(ordtablerows[ordValueLevel1]['childlist']):
for shareValue in tablerows:
if ordValueLevel2 in shareValue['level2']:
if len(ordtablerows[ordValueLevel1][ordValueLevel2]['childlist'])==0:
ord1Short = re.sub('\[(.*)\]','',ordValueLevel1).rstrip().lower()
ord2Short = re.sub('\[(.*)\]', '', ordValueLevel2).rstrip().lower()
link = "?jobtype=%s&display_limit=100&gshare=%s"%(ord1Short,ord2Short)
shareValue['link'] = link
shareValue['level'] = 'level2'
shareValue['gshare'] = ord2Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if 'level' in shareValue:
add_resources(ord2Short, newTablesRow, resources_list, shareValue['level'])
break
for ordValueLevel3 in sorted(ordtablerows[ordValueLevel1][ordValueLevel2]['childlist']):
for shareValue in tablerows:
if ordValueLevel3 in shareValue['level3']:
if len(ordtablerows[ordValueLevel1][ordValueLevel2]['childlist']) > 0:
ord1Short = re.sub('\[(.*)\]', '', ordValueLevel1).rstrip().lower()
ord3Short = re.sub('\[(.*)\]', '', ordValueLevel3).rstrip().lower()
link = "?jobtype=%s&display_limit=100&gshare=%s" % (ord1Short, ord3Short)
shareValue['link'] = link
shareValue['level'] = 'level3'
shareValue['gshare'] = ord3Short.replace(' ', '_')
newTablesRow.append(shareValue)
tablerows.remove(shareValue)
if 'level' in shareValue:
add_resources(ord3Short, newTablesRow, resources_list, shareValue['level'])
break
tablerows = newTablesRow
del request.session['TFIRST']
del request.session['TLAST']
##self monitor
endSelfMonitor(request)
if (not (('HTTP_ACCEPT' in request.META) and (request.META.get('HTTP_ACCEPT') in ('application/json'))) and (
'json' not in request.session['requestParams'])):
data = {
'request': request,
'viewParams': request.session['viewParams'],
'requestParams': request.session['requestParams'],
'globalshares': gs,
'xurl': extensibleURL(request),
'gsPlotData':gsPlotData,
'tablerows':tablerows,
'built': datetime.now().strftime("%H:%M:%S"),
}
response = render_to_response('globalshares.html', data, content_type='text/html')
setCacheEntry(request, "globalshares", json.dumps(data, cls=DateEncoder), 60 * 20)
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
return response
else:
return HttpResponse(DecimalEncoder().encode(gs), content_type='text/html')
def compareJobs(request):
valid, response = initRequest(request)
if not valid: return response
pandaidstr = None
if 'pandaid' in request.session['requestParams']:
pandaidstr = request.session['requestParams']['pandaid'].split('|')
else:
query = {}
query['userid'] = request.user.id
query['object'] = 'job'
try:
jobsComparison = ObjectsComparison.objects.get(**query)
pandaidstr = json.loads(jobsComparison.comparisonlist)
except ObjectsComparison.DoesNotExist:
pandaidstr = None
if not pandaidstr:
return render_to_response('errorPage.html', {'errormessage': 'No pandaids for comparison provided'}, content_type='text/html')
pandaids = []
for pid in pandaidstr:
try:
pid = int(pid)
pandaids.append(pid)
except:
pass
maxNJobs = 5
if len(pandaids) > maxNJobs:
pandaids = pandaids[:maxNJobs]
jobInfoJSON = []
# Looking for a job in cache
pandaidsToBeLoad = []
for pandaid in pandaids:
data = getCacheEntry(request, "compareJob_" + str(pandaid), isData=True)
# data = None
if data is not None:
jobInfoJSON.append(json.loads(data))
else:
pandaidsToBeLoad.append(pandaid)
#Loading jobs info in parallel
nprocesses = maxNJobs
if len(pandaidsToBeLoad) > 0:
url_params = [('?json=1&pandaid=' + str(pid)) for pid in pandaidsToBeLoad]
pool = multiprocessing.Pool(processes=nprocesses)
jobInfoJSON.extend(pool.map(job_info_getter, url_params))
pool.close()
pool.join()
#Put loaded jobs info to cache
for job in jobInfoJSON:
setCacheEntry(request, "compareJob_" + str(job.keys()[0]),
json.dumps(job.values()[0], cls=DateEncoder), 60 * 30, isData=True)
compareParamNames = {'produsername': '******', 'reqid': 'Request ID', 'jeditaskid': 'Task ID', 'jobstatus': 'Status',
'attemptnr': 'Attempt', 'creationtime': 'Created', 'waittime': 'Time to start', 'duration': 'Duration',
'modificationtime': 'Modified', 'cloud': 'Cloud', 'computingsite': 'Site', 'currentpriority': 'Priority',
'jobname': 'Name', 'processingtype': 'Type', 'transformation': 'Transformation', 'proddblock': 'Input',
'destinationdblock': 'Output', 'jobsetid': 'Jobset ID', 'batchid': 'Batch ID', 'eventservice': 'Event Service'}
compareParams = ['produsername', 'reqid', 'jeditaskid', 'jobstatus', 'attemptnr','creationtime', 'waittime', 'duration',
'modificationtime', 'cloud', 'computingsite','currentpriority',
'jobname', 'processingtype', 'transformation','proddblock','destinationdblock', 'jobsetid', 'batchid','eventservice']
###Excluded params because of too long values###
excludedParams = ['metadata', 'metastruct']
jobsComparisonMain = []
for param in compareParams:
row = [{'paramname': compareParamNames[param]}]
for jobd in jobInfoJSON:
job = jobd['job']
if param in job:
row.append({'value': job[param]})
else:
row.append({'value': '-'})
if len(set([d['value'] for d in row if 'value' in d])) == 1:
row[0]['mark'] = 'equal'
jobsComparisonMain.append(row)
all_params = []
for jobd in jobInfoJSON:
all_params.extend(list(jobd['job'].keys()))
all_params = sorted(set(all_params))
jobsComparisonAll = []
for param in all_params:
if param not in excludedParams:
row = [{'paramname': param}]
for jobd in jobInfoJSON:
job = jobd['job']
if param in job and job[param] is not None:
row.append({'value': job[param]})
else:
row.append({'value': '-'})
if len(set([d['value'] for d in row if 'value' in d])) == 1:
row[0]['mark'] = 'equal'
jobsComparisonAll.append(row)
xurl = extensibleURL(request)
data = {
'request': request,
'viewParams': request.session['viewParams'],
'requestParams': request.session['requestParams'],
'url': request.path,
'jobsComparisonMain': jobsComparisonMain,
'jobsComparisonAll': jobsComparisonAll,
'pandaids': pandaids,
'xurl': xurl,
'built': datetime.now().strftime("%H:%M:%S"),
}
##self monitor
endSelfMonitor(request)
response = render_to_response('compareJobs.html', data, content_type='text/html')
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
return response
def artJobs(request):
valid, response = initRequest(request)
if not valid: return response
# Here we try to get cached data
data = getCacheEntry(request, "artJobs")
# data = None
if data is not None:
data = json.loads(data)
data['request'] = request
if 'ntaglist' in data:
if len(data['ntaglist']) > 0:
ntags = []
for ntag in data['ntaglist']:
try:
ntags.append(datetime.strptime(ntag, artdateformat))
except:
pass
if len(ntags) > 1 and 'requestParams' in data:
data['requestParams']['ntag_from'] = min(ntags)
data['requestParams']['ntag_to'] = max(ntags)
elif len(ntags) == 1:
data['requestParams']['ntag'] = ntags[0]
response = render_to_response('artJobs.html', data, content_type='text/html')
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
endSelfMonitor(request)
return response
query = setupView(request, 'job')
cur = connection.cursor()
if datetime.strptime(query['ntag_from'], '%Y-%m-%d') < datetime.strptime('2018-03-20', '%Y-%m-%d'):
cur.execute("SELECT * FROM table(ATLAS_PANDABIGMON.ARTTESTS('%s','%s','%s'))" % (query['ntag_from'], query['ntag_to'], query['strcondition']))
else:
cur.execute("SELECT * FROM table(ATLAS_PANDABIGMON.ARTTESTS_1('%s','%s','%s'))" % (query['ntag_from'], query['ntag_to'], query['strcondition']))
jobs = cur.fetchall()
cur.close()
artJobsNames = ['taskid','package', 'branch', 'ntag', 'nightly_tag', 'testname', 'jobstatus', 'origpandaid', 'computingsite', 'endtime', 'starttime' , 'maxvmem', 'cpuconsumptiontime', 'guid', 'scope', 'lfn', 'taskstatus', 'taskmodificationtime', 'jobmodificationtime', 'result']
jobs = [dict(zip(artJobsNames, row)) for row in jobs]
# i=0
# for job in jobs:
# i+=1
# print 'registering %i out of %i jobs' % (i, len(jobs))
# x = ArtTest(job['origpandaid'], job['testname'], job['branch'].split('/')[0], job['branch'].split('/')[1],job['branch'].split('/')[2], job['package'], job['nightly_tag'])
# if x.registerArtTest():
# print '%i job registered sucessfully out of %i' % (i, len(jobs))
ntagslist=list(sorted(set([x['ntag'] for x in jobs])))
jeditaskids = list(sorted(set([x['taskid'] for x in jobs])))
artjobsdict={}
if not 'view' in request.session['requestParams'] or (
'view' in request.session['requestParams'] and request.session['requestParams']['view'] == 'packages'):
for job in jobs:
if job['package'] not in artjobsdict.keys():
artjobsdict[job['package']] = {}
if job['branch'] not in artjobsdict[job['package']].keys():
artjobsdict[job['package']][job['branch']] = {}
if job['testname'] not in artjobsdict[job['package']][job['branch']].keys():
artjobsdict[job['package']][job['branch']][job['testname']] = {}
for n in ntagslist:
artjobsdict[job['package']][job['branch']][job['testname']][n.strftime(artdateformat)] = {}
artjobsdict[job['package']][job['branch']][job['testname']][n.strftime(artdateformat)]['ntag_hf'] = n.strftime(humandateformat)
artjobsdict[job['package']][job['branch']][job['testname']][n.strftime(artdateformat)]['jobs'] = []
if job['ntag'].strftime(artdateformat) in artjobsdict[job['package']][job['branch']][job['testname']]:
jobdict = {}
jobdict['jobstatus'] = job['jobstatus']
jobdict['origpandaid'] = job['origpandaid']
jobdict['linktext'] = job['branch'] + '/' + job['nightly_tag'] + '/' + job['package'] + '/' + job['testname'][:-3]
jobdict['computingsite'] = job['computingsite']
jobdict['guid'] = job['guid']
jobdict['scope'] = job['scope']
jobdict['lfn'] = job['lfn']
jobdict['jeditaskid'] = job['taskid']
jobdict['maxvmem'] = round(job['maxvmem']*1.0/1000,1) if job['maxvmem'] is not None else '---'
jobdict['cpuconsumptiontime'] = job['cpuconsumptiontime'] if job['jobstatus'] in ('finished', 'failed') else '---'
if job['jobstatus'] in ('finished', 'failed'):
jobdict['duration'] = job['endtime'] - job['starttime']
else:
jobdict['duration'] = str(datetime.now() - job['starttime']).split('.')[0] if job['starttime'] is not None else "---"
try:
jobdict['tarindex'] = int(re.search('.([0-9]{6}).log.', job['lfn']).group(1))
except:
jobdict['tarindex'] = ''
finalresult, testexitcode, subresults, testdirectory = getFinalResult(job)
jobdict['finalresult'] = finalresult
jobdict['testexitcode'] = testexitcode
jobdict['testresult'] = subresults
jobdict['testdirectory'] = testdirectory
artjobsdict[job['package']][job['branch']][job['testname']][job['ntag'].strftime(artdateformat)]['jobs'].append(jobdict)
elif 'view' in request.session['requestParams'] and request.session['requestParams']['view'] == 'branches':
for job in jobs:
if job['branch'] not in artjobsdict.keys():
artjobsdict[job['branch']] = {}
if job['package'] not in artjobsdict[job['branch']].keys():
artjobsdict[job['branch']][job['package']] = {}
if job['testname'] not in artjobsdict[job['branch']][job['package']].keys():
artjobsdict[job['branch']][job['package']][job['testname']] = {}
for n in ntagslist:
artjobsdict[job['branch']][job['package']][job['testname']][n.strftime(artdateformat)] = {}
artjobsdict[job['branch']][job['package']][job['testname']][n.strftime(artdateformat)]['ntag_hf'] = n.strftime(humandateformat)
artjobsdict[job['branch']][job['package']][job['testname']][n.strftime(artdateformat)]['jobs'] = []
if job['ntag'].strftime(artdateformat) in artjobsdict[job['branch']][job['package']][job['testname']]:
jobdict = {}
jobdict['jobstatus'] = job['jobstatus']
jobdict['origpandaid'] = job['origpandaid']
jobdict['linktext'] = job['branch'] + '/' + job['nightly_tag'] + '/' + job['package'] + '/' + job['testname'][:-3]
jobdict['computingsite'] = job['computingsite']
jobdict['guid'] = job['guid']
jobdict['scope'] = job['scope']
jobdict['lfn'] = job['lfn']
jobdict['jeditaskid'] = job['taskid']
jobdict['maxvmem'] = round(job['maxvmem'] * 1.0 / 1000, 1) if job['maxvmem'] is not None else '---'
jobdict['cpuconsumptiontime'] = job['cpuconsumptiontime'] if job['jobstatus'] in (
'finished', 'failed') else '---'
if job['jobstatus'] in ('finished', 'failed'):
jobdict['duration'] = job['endtime'] - job['starttime']
else:
jobdict['duration'] = str(datetime.now() - job['starttime']).split('.')[0] if job['starttime'] is not None else "---"
try:
jobdict['tarindex'] = int(re.search('.([0-9]{6}).log.', job['lfn']).group(1))
except:
jobdict['tarindex'] = ''
finalresult, testexitcode, subresults, testdirectory = getFinalResult(job)
jobdict['finalresult'] = finalresult
jobdict['testexitcode'] = testexitcode
jobdict['testresult'] = subresults
jobdict['testdirectory'] = testdirectory
artjobsdict[job['branch']][job['package']][job['testname']][job['ntag'].strftime(artdateformat)]['jobs'].append(jobdict)
xurl = extensibleURL(request)
noviewurl = removeParam(xurl, 'view', mode='extensible')
if (('HTTP_ACCEPT' in request.META) and (request.META.get('HTTP_ACCEPT') in ('text/json', 'application/json'))) or (
'json' in request.session['requestParams']):
data = {
'artjobs': artjobsdict,
}
dump = json.dumps(data, cls=DateEncoder)
return HttpResponse(dump, content_type='text/html')
else:
data = {
'requestParams': request.session['requestParams'],
'viewParams': request.session['viewParams'],
'artjobs': artjobsdict,
'noviewurl': noviewurl,
'ntaglist': [ntag.strftime(artdateformat) for ntag in ntagslist],
'taskids' : jeditaskids,
}
setCacheEntry(request, "artJobs", json.dumps(data, cls=DateEncoder), 60 * cache_timeout)
response = render_to_response('artJobs.html', data, content_type='text/html')
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
endSelfMonitor(request)
return response
def artTasks(request):
valid, response = initRequest(request)
query = setupView(request, 'job')
# Here we try to get cached data
data = getCacheEntry(request, "artTasks")
# data = None
if data is not None:
data = json.loads(data)
data['request'] = request
if 'ntaglist' in data:
if len(data['ntaglist']) > 0:
ntags = []
for ntag in data['ntaglist']:
try:
ntags.append(datetime.strptime(ntag, artdateformat))
except:
pass
if len(ntags) > 1 and 'requestParams' in data:
data['requestParams']['ntag_from'] = min(ntags)
data['requestParams']['ntag_to'] = max(ntags)
elif len(ntags) == 1:
data['requestParams']['ntag'] = ntags[0]
response = render_to_response('artTasks.html', data, content_type='text/html')
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
endSelfMonitor(request)
return response
cur = connection.cursor()
if datetime.strptime(query['ntag_from'], '%Y-%m-%d') < datetime.strptime('2018-03-20', '%Y-%m-%d'):
query_raw = """SELECT package, branch, ntag, taskid, status, result FROM table(ATLAS_PANDABIGMON.ARTTESTS('%s','%s','%s'))""" % (query['ntag_from'], query['ntag_to'], query['strcondition'])
else:
query_raw = """SELECT package, branch, ntag, taskid, status, result FROM table(ATLAS_PANDABIGMON.ARTTESTS_1('%s','%s','%s'))""" % (query['ntag_from'], query['ntag_to'], query['strcondition'])
cur.execute(query_raw)
tasks_raw = cur.fetchall()
cur.close()
artJobs = ['package', 'branch', 'ntag', 'task_id', 'jobstatus', 'result']
jobs = [dict(zip(artJobs, row)) for row in tasks_raw]
# tasks = ARTTasks.objects.filter(**query).values('package','branch','task_id', 'ntag', 'nfilesfinished', 'nfilesfailed')
ntagslist = list(sorted(set([x['ntag'] for x in jobs])))
statestocount = ['finished', 'failed', 'active']
arttasksdict = {}
if not 'view' in request.session['requestParams'] or ('view' in request.session['requestParams'] and request.session['requestParams']['view'] == 'packages'):
for job in jobs:
if job['package'] not in arttasksdict.keys():
arttasksdict[job['package']] = {}
if job['branch'] not in arttasksdict[job['package']].keys():
arttasksdict[job['package']][job['branch']] = {}
for n in ntagslist:
arttasksdict[job['package']][job['branch']][n.strftime(artdateformat)] = {}
arttasksdict[job['package']][job['branch']][n.strftime(artdateformat)]['ntag_hf'] = n.strftime(humandateformat)
for state in statestocount:
arttasksdict[job['package']][job['branch']][n.strftime(artdateformat)][state] = 0
if job['ntag'].strftime(artdateformat) in arttasksdict[job['package']][job['branch']]:
finalresult, testexitcode, subresults, testdirectory = getFinalResult(job)
arttasksdict[job['package']][job['branch']][job['ntag'].strftime(artdateformat)][finalresult] += 1
elif 'view' in request.session['requestParams'] and request.session['requestParams']['view'] == 'branches':
for job in jobs:
if job['branch'] not in arttasksdict.keys():
arttasksdict[job['branch']] = {}
if job['package'] not in arttasksdict[job['branch']].keys():
arttasksdict[job['branch']][job['package']] = {}
for n in ntagslist:
arttasksdict[job['branch']][job['package']][n.strftime(artdateformat)] = {}
arttasksdict[job['branch']][job['package']][n.strftime(artdateformat)]['ntag_hf'] = n.strftime(humandateformat)
for state in statestocount:
arttasksdict[job['branch']][job['package']][n.strftime(artdateformat)][state] = 0
if job['ntag'].strftime(artdateformat) in arttasksdict[job['branch']][job['package']]:
finalresult, testexitcode, subresults, testdirectory = getFinalResult(job)
arttasksdict[job['branch']][job['package']][job['ntag'].strftime(artdateformat)][finalresult] += 1
xurl = extensibleURL(request)
noviewurl = removeParam(xurl, 'view', mode='extensible')
if (('HTTP_ACCEPT' in request.META) and (request.META.get('HTTP_ACCEPT') in ('text/json', 'application/json'))) or (
'json' in request.session['requestParams']):
data = {
'arttasks' : arttasksdict,
}
dump = json.dumps(data, cls=DateEncoder)
return HttpResponse(dump, content_type='text/html')
else:
data = {
'requestParams': request.session['requestParams'],
'viewParams': request.session['viewParams'],
'arttasks' : arttasksdict,
'noviewurl': noviewurl,
'ntaglist': [ntag.strftime(artdateformat) for ntag in ntagslist],
}
setCacheEntry(request, "artTasks", json.dumps(data, cls=DateEncoder), 60 * cache_timeout)
response = render_to_response('artTasks.html', data, content_type='text/html')
patch_response_headers(response, cache_timeout=request.session['max_age_minutes'] * 60)
endSelfMonitor(request)
return response