def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
# massage data
#print "L1", len(wlcg_data)
t_dataprocess_start = time.time()
wlcg_records = dataprocess.rowsToDicts(wlcg_data)
wlcg_records = dataprocess.addMissingScaleValues(wlcg_records)
wlcg_records = dataprocess.collapseFields(wlcg_records, self.collapse)
if self.tier_based:
wlcg_records = dataprocess.tierMergeSplit(wlcg_records, self.tier_mapping, self.tier_shares, self.default_tier)
if self.split != dataprocess.TIER:
wlcg_records = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, ) )
# information on ops vo does not add any value
wlcg_records = [ rec for rec in wlcg_records if rec[dataprocess.VO_NAME] != 'ops' ]
wlcg_records = dataprocess.addEffiencyProperty(wlcg_records)
wlcg_records = dataprocess.addEquivalentProperties(wlcg_records, days)
sk = lambda key : dataprocess.sortKey(key, field_order=self.columns)
if self.split is None:
wlcg_records = sorted(wlcg_records, key=sk)
else:
split_records = dataprocess.splitRecords(wlcg_records, dataprocess.TIER)
for split_attr, records in split_records.items():
split_records[split_attr] = sorted(records, key=sk)
#print "L2", len(wlcg_records)
t_dataprocess = time.time() - t_dataprocess_start
if self.split is None:
table_content = self.createTable(wlcg_records, self.columns)
else:
table_content = ''
for split_attr, records in split_records.items():
table = self.createTable(records, self.columns)
table_content += html.createParagraph(split_attr) + table + html.SECTION_BREAK
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG %s view' % self.path
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option)
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
# massage data
#print "L1", len(wlcg_data)
t_dataprocess_start = time.time()
wlcg_records = dataprocess.rowsToDicts(wlcg_data)
wlcg_records = dataprocess.addMissingScaleValues(wlcg_records)
wlcg_records = dataprocess.collapseFields(wlcg_records, self.collapse)
if self.tier_based:
wlcg_records = dataprocess.tierMergeSplit(wlcg_records, self.tier_mapping, self.tier_shares, self.default_tier)
if self.split != dataprocess.TIER:
wlcg_records = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, ) )
# information on ops vo does not add any value
wlcg_records = [ rec for rec in wlcg_records if rec[dataprocess.VO_NAME] != 'ops' ]
wlcg_records = dataprocess.addEffiencyProperty(wlcg_records)
wlcg_records = dataprocess.addEquivalentProperties(wlcg_records, days)
sk = lambda key : dataprocess.sortKey(key, field_order=self.columns)
if self.split is None:
wlcg_records = sorted(wlcg_records, key=sk)
else:
split_records = dataprocess.splitRecords(wlcg_records, dataprocess.TIER)
for split_attr, records in split_records.items():
split_records[split_attr] = sorted(records, key=sk)
#print "L2", len(wlcg_records)
t_dataprocess = time.time() - t_dataprocess_start
if self.split is None:
table_content = self.createTable(wlcg_records, self.columns)
else:
table_content = ''
for split_attr, records in split_records.items():
table = self.createTable(records, self.columns)
table_content += html.createParagraph(split_attr) + table + html.SECTION_BREAK
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG %s view' % self.path
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option)
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
t_dataprocess_start = time.time()
wlcg_records = wlcg.rowsToDicts(wlcg_data, self.columns)
wlcg_records = _changeUnits(wlcg_records)
sk = lambda key : _sortKey(key, field_order=self.columns)
if self.split is None:
wlcg_records = self.sort(wlcg_records, key=sk)
else:
split_records = _splitRecords(wlcg_records, self.split)
for split_attr, records in split_records.items():
split_records[split_attr] = self.sort(records, key=sk)
t_dataprocess = time.time() - t_dataprocess_start
if self.split is None:
table_content = self.createTable(wlcg_records, self.columns)
else:
table_content = ''
columns = [ c for c in self.columns if c != self.split and c not in self.invisible_columns ]
for split_attr, records in split_records.items():
table = self.createTable(records, columns)
table_content += html.createParagraph(split_attr) + table + html.SECTION_BREAK
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG %s view' % self.path
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option)
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, db_rows, request, date, media, t_query_start):
t_query = time.time() - t_query_start
t_dataprocess_start = time.time()
records = self.buildRecords(db_rows)
# remove infomration from certain groups, add they not add any value
records = [ rec for rec in records if rec['group'] not in ('dteam', 'behrmann') ]
if media == 'disk':
records = [ rec for rec in records if rec['media'] == 'disk' ]
elif media == 'tape':
records = [ rec for rec in records if rec['media'] == 'tape' ]
elif media == 'all':
pass
else:
return self.renderErrorPage('Invalid media selection', request)
# get top level domains and sites per country
hosts = set( [ rec['site'] for rec in records ] )
tld_groups = {}
country_sites = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# get groups
groups = set( [ rec['group'] for rec in records ] )
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate totals per site / group
site_group_totals = {}
for rec in records:
site, group, rcu = rec['site'], rec['group'], rec['rcu']
key = (site, group)
site_group_totals[key] = site_group_totals.get(key, 0) + rcu
# calculate total per site
site_totals = {}
for rec in records:
site, rcu = rec['site'], rec['rcu']
site_totals[site] = site_totals.get(site,0) + rcu
# calculate total per group / country
group_country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
key = (country, rec['group'])
group_country_totals[key] = group_country_totals.get(key, 0) + rec['rcu']
# calculate total per country
country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
country_totals[country] = country_totals.get(country,0) + rec['rcu']
# calculate total per group
group_totals = {}
for rec in records:
group_totals[rec['group']] = group_totals.get(rec['group'],0) + rec['rcu']
# calculate total
total = sum( [ rec['rcu'] for rec in records ] )
totals = []
# put all calculated records together and add equivalents
for (site, group), rcu in site_group_totals.items():
totals.append( { 'site': site, 'group': group, 'rcu': rcu } )
for site, rcu in site_totals.items():
totals.append( { 'site': site, 'group': TOTAL, 'rcu': rcu } )
for (country, group), rcu in group_country_totals.items():
totals.append( { 'site': country, 'group': group, 'rcu': rcu } )
for country, rcu in country_totals.items():
totals.append( { 'site': country, 'group': TOTAL, 'rcu': rcu } )
for group, rcu in group_totals.items():
totals.append( { 'site': TIER_TOTAL, 'group': group, 'rcu': rcu } )
totals.append( { 'site': TIER_TOTAL, 'group': TOTAL, 'rcu': total } )
# create table
columns = sorted(groups)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += sorted(tld_groups[tld])
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
elements = []
for row in row_names:
for col in columns:
for rec in totals:
if rec['site'] == row and rec['group'] == col:
value = rec['rcu']
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
t_dataprocess = time.time() - t_dataprocess_start
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
title = 'WLCG storage view'
media_options = [ ( 'all', 'Disk and Tape') , ( 'disk', 'Disk only' ), ( 'tape', 'Tape only' ) ]
media_buttons = html.createRadioButtons('media', media_options, checked_value=media)
month_options = dateform.generateMonthFormOptions()
selector = html.createSelector('Month', 'date', month_options, date)
selector_form = html.createSelectorForm(self.path, [ selector ], media_buttons )
date_text = html.createParagraph('Date: %s' % (date))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(date_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, unit, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
t_dataprocess_start = time.time()
wlcg_records = dataprocess.rowsToDicts(wlcg_data)
# information on ops and dteam vo does not add any value
wlcg_records = [ rec for rec in wlcg_records if rec[dataprocess.VO_NAME] not in ('dteam', 'ops') ]
# massage data
wlcg_records = dataprocess.addMissingScaleValues(wlcg_records)
wlcg_records = dataprocess.collapseFields(wlcg_records, self.collapse)
wlcg_records = dataprocess.tierMergeSplit(wlcg_records, self.tier_mapping, self.tier_shares, self.default_tier)
# role must be collapsed after split in order for the tier split to function
wlcg_records = dataprocess.collapseFields(wlcg_records, [ dataprocess.VO_ROLE ] )
sort_key = lambda key : dataprocess.sortKey(key, field_order=[ dataprocess.HOST] )
split_records = dataprocess.splitRecords(wlcg_records, dataprocess.TIER)
for split_attr, records in split_records.items():
split_records[split_attr] = sorted(records, key=sort_key)
t_dataprocess = time.time() - t_dataprocess_start
# get tld groups
hosts = set( [ rec[dataprocess.HOST] for rec in wlcg_records ] )
tld_groups = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# create composite to-tier names
vo_tiers = set()
for rec in wlcg_records:
tl = 't1' if 'T1' in rec[dataprocess.TIER] else 't2'
vt = tl + '-' + rec[dataprocess.VO_NAME]
rec[dataprocess.VO_NAME] = vt
del rec[dataprocess.TIER] # same as collapsing afterwards
vo_tiers.add(vt)
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate total per site
site_totals = dataprocess.collapseFields(wlcg_records, ( dataprocess.VO_NAME, ) )
for r in site_totals:
r[dataprocess.VO_NAME] = TOTAL
# calculate total per country-tier
country_tier_totals = [ r.copy() for r in wlcg_records ]
for rec in country_tier_totals:
rec[dataprocess.HOST] = rec[dataprocess.HOST].split('.')[-1].upper() + '-TOTAL'
rec[dataprocess.USER] = 'FAKE'
country_tier_totals = dataprocess.collapseFields(country_tier_totals, ( dataprocess.USER, ) )
# calculate total per country
country_totals = dataprocess.collapseFields(country_tier_totals, ( dataprocess.VO_NAME, ) )
for rec in country_totals:
rec[dataprocess.VO_NAME] = TOTAL
# calculate total per tier-vo
tier_vo_totals = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, ) )
for r in tier_vo_totals:
r[dataprocess.HOST] = TIER_TOTAL
# calculate total
total = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, dataprocess.VO_NAME ) )
assert len(total) in (0,1), 'Records did not collapse into a single record when calculating grand total'
if len(total) == 0:
total = [ { dataprocess.CPU_TIME : 0, dataprocess.WALL_TIME : 0, dataprocess.KSI2K_CPU_TIME : 0, dataprocess.KSI2K_WALL_TIME : 0 } ]
total_record = total[0]
total_record[dataprocess.HOST] = TIER_TOTAL
total_record[dataprocess.VO_NAME] = TOTAL
# put all calculated records together and add equivalents
wlcg_records += site_totals
wlcg_records += country_tier_totals
wlcg_records += country_totals
wlcg_records += tier_vo_totals
wlcg_records += [ total_record ]
wlcg_records = dataprocess.addEquivalentProperties(wlcg_records, days)
# create table
columns = sorted(vo_tiers)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += tld_groups[tld]
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
unit_extractor = WLCG_UNIT_MAPPING.get(unit, WLCG_UNIT_MAPPING_DEFAULT)
elements = []
for row in row_names:
for col in columns:
for rec in wlcg_records:
if rec[dataprocess.HOST] == row and rec[dataprocess.VO_NAME] == col:
value = _formatValue( unit_extractor(rec) )
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
# render page
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG oversight view'
unit_options = [ ( 'ksi2k-ne', 'KSI2K Node Equivalents (default)') , ( 'hs06-ne', 'HS06 Node Equivalents' ),
( 'ksi2k-wallhours', 'KSI2K Walltime Hours' ) , ( 'hs06-wallhours', 'HS06 Walltime hours') ]
unit_buttons = html.createRadioButtons('unit', unit_options, checked_value=unit)
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option, unit_buttons)
quarters = dateform.generateFormQuarters()
quarter_links = []
for q in quarters:
year, quart = dateform.parseQuarter(q)
sd, ed = dateform.quarterStartEndDates(year, quart)
quarter_links.append(html.createLink('%s?startdate=%s&enddate=%s' % (self.path, sd, ed), q ) )
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph('Quarters: \n ' + (' ' + html.NBSP).join(quarter_links) ) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderMachineView(self, results, request):
machine_manifest = results[0][1][0]
jobs_per_day = results[1][1]
top_projects = results[2][1]
top_users = results[3][1]
first_record_registration = machine_manifest[0]
last_record_registration = machine_manifest[1]
first_job_start = machine_manifest[2]
last_job_termination = machine_manifest[3]
distinct_users = machine_manifest[4]
distinct_projects = machine_manifest[5]
n_jobs = machine_manifest[6]
e_batches = [ e[0] for e in jobs_per_day ] # dates
e_groups = [ 'jobs' ]
e_matrix = dict( [ ((e[0], e_groups[0]), e[1]) for e in jobs_per_day ] )
executed_table = htmltable.createHTMLTable(e_matrix, e_batches, e_groups, base_indent=4)
stats_batches = [ 'Walltime days', 'Efficiency', 'Number of jobs' ]
p_groups = [ e[0] for e in top_projects ] # user list
p_dict_elements = []
p_dict_elements += [ ((stats_batches[0], p[0]), p[1]) for p in top_projects ]
p_dict_elements += [ ((stats_batches[1], p[0]), p[2]) for p in top_projects ]
p_dict_elements += [ ((stats_batches[2], p[0]), p[3]) for p in top_projects ]
p_matrix = dict(p_dict_elements)
project_table = htmltable.createHTMLTable(p_matrix, stats_batches, p_groups, base_indent=4)
u_groups = [ e[0] for e in top_users ] # user list
u_dict_elements = []
u_dict_elements += [ ((stats_batches[0], u[0]), u[1]) for u in top_users ]
u_dict_elements += [ ((stats_batches[1], u[0]), u[2]) for u in top_users ]
u_dict_elements += [ ((stats_batches[2], u[0]), u[3]) for u in top_users ]
u_matrix = dict(u_dict_elements)
user_table = htmltable.createHTMLTable(u_matrix, stats_batches, u_groups, base_indent=4)
title = 'Machine view for %s' % self.machine_name
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
# generate year-month options one year back
month_options = ['']
gmt = time.gmtime()
for i in range(gmt.tm_mon-12, gmt.tm_mon+1):
if i <= 0:
month_options.append('%i-%02d' % (gmt.tm_year - 1, 12 + i) )
elif i > 0:
month_options.append('%i-%02d' % (gmt.tm_year, i) )
sel1 = html.createSelector('Start month', 'startdate', month_options, start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options, end_date_option)
selector_form = html.createSelectorForm(self.machine_name, [sel1, sel2] )
if start_date_option or end_date_option:
range_text = 'selected date range'
else:
range_text = 'current month'
# create page
request.write(html.HTML_VIEWBASE_HEADER % {'title': title})
request.write( html.createTitle(title) )
request.write('\n' + selector_form + '\n')
if not (start_date_option or end_date_option):
# skip manifest / inserts if date range is selected
request.write( html.createSectionTitle('Manifest') )
request.write( html.createParagraph('First record registration: %s' % first_record_registration) )
request.write( html.createParagraph('Last record registration: %s' % last_record_registration) )
request.write( html.createParagraph('First job start: %s' % first_job_start) )
request.write( html.createParagraph('Last job termination: %s' % last_job_termination) )
request.write( html.createParagraph('Distinct users: %s' % distinct_users) )
request.write( html.createParagraph('Distinct projects: %s' % distinct_projects) )
request.write( html.createParagraph('Number of jobs: %s' % n_jobs) )
request.write( html.SECTION_BREAK )
request.write( html.createSectionTitle('Executed jobs in the last ten days') )
request.write( executed_table )
request.write( html.SECTION_BREAK)
request.write( html.createSectionTitle('Top 10 projects for the %s' % range_text) )
request.write( project_table )
request.write( html.SECTION_BREAK )
request.write( html.createSectionTitle('Top 20 users for the %s' % range_text) )
request.write( user_table )
request.write( html.P + '\n' )
request.write(html.HTML_VIEWBASE_FOOTER)
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, db_rows, request, date, media, t_query_start):
t_query = time.time() - t_query_start
t_dataprocess_start = time.time()
records = self.buildRecords(db_rows)
# remove infomration from certain groups, add they not add any value
records = [ rec for rec in records if rec['group'] not in ('dteam', 'behrmann') ]
if media == 'disk':
records = [ rec for rec in records if rec['media'] == 'disk' ]
elif media == 'tape':
records = [ rec for rec in records if rec['media'] == 'tape' ]
elif media == 'all':
pass
else:
return self.renderErrorPage('Invalid media selection', request)
# get top level domains and sites per country
hosts = set( [ rec['site'] for rec in records ] )
tld_groups = {}
country_sites = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# get groups
groups = set( [ rec['group'] for rec in records ] )
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate totals per site / group
site_group_totals = {}
for rec in records:
site, group, rcu = rec['site'], rec['group'], rec['rcu']
key = (site, group)
site_group_totals[key] = site_group_totals.get(key, 0) + rcu
# calculate total per site
site_totals = {}
for rec in records:
site, rcu = rec['site'], rec['rcu']
site_totals[site] = site_totals.get(site,0) + rcu
# calculate total per group / country
group_country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
key = (country, rec['group'])
group_country_totals[key] = group_country_totals.get(key, 0) + rec['rcu']
# calculate total per country
country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
country_totals[country] = country_totals.get(country,0) + rec['rcu']
# calculate total per group
group_totals = {}
for rec in records:
group_totals[rec['group']] = group_totals.get(rec['group'],0) + rec['rcu']
# calculate total
total = sum( [ rec['rcu'] for rec in records ] )
totals = []
# put all calculated records together and add equivalents
for (site, group), rcu in site_group_totals.items():
totals.append( { 'site': site, 'group': group, 'rcu': rcu } )
for site, rcu in site_totals.items():
totals.append( { 'site': site, 'group': TOTAL, 'rcu': rcu } )
for (country, group), rcu in group_country_totals.items():
totals.append( { 'site': country, 'group': group, 'rcu': rcu } )
for country, rcu in country_totals.items():
totals.append( { 'site': country, 'group': TOTAL, 'rcu': rcu } )
for group, rcu in group_totals.items():
totals.append( { 'site': TIER_TOTAL, 'group': group, 'rcu': rcu } )
totals.append( { 'site': TIER_TOTAL, 'group': TOTAL, 'rcu': total } )
# create table
columns = sorted(groups)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += sorted(tld_groups[tld])
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
elements = []
for row in row_names:
for col in columns:
for rec in totals:
if rec['site'] == row and rec['group'] == col:
value = rec['rcu']
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
t_dataprocess = time.time() - t_dataprocess_start
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
title = 'WLCG storage view'
media_options = [ ( 'all', 'Disk and Tape') , ( 'disk', 'Disk only' ), ( 'tape', 'Tape only' ) ]
media_buttons = html.createRadioButtons('media', media_options, checked_value=media)
month_options = dateform.generateMonthFormOptions()
selector = html.createSelector('Month', 'date', month_options, date)
selector_form = html.createSelectorForm(self.path, [ selector ], media_buttons )
date_text = html.createParagraph('Date: %s' % (date))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(date_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, unit, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
t_dataprocess_start = time.time()
wlcg_records = dataprocess.rowsToDicts(wlcg_data)
# information on ops and dteam vo does not add any value
wlcg_records = [ rec for rec in wlcg_records if rec[dataprocess.VO_NAME] not in ('dteam', 'ops') ]
# massage data
wlcg_records = dataprocess.addMissingScaleValues(wlcg_records)
wlcg_records = dataprocess.collapseFields(wlcg_records, self.collapse)
wlcg_records = dataprocess.tierMergeSplit(wlcg_records, self.tier_mapping, self.tier_shares, self.default_tier)
# role must be collapsed after split in order for the tier split to function
wlcg_records = dataprocess.collapseFields(wlcg_records, [ dataprocess.VO_ROLE ] )
sort_key = lambda key : dataprocess.sortKey(key, field_order=[ dataprocess.HOST] )
split_records = dataprocess.splitRecords(wlcg_records, dataprocess.TIER)
for split_attr, records in split_records.items():
split_records[split_attr] = sorted(records, key=sort_key)
t_dataprocess = time.time() - t_dataprocess_start
# get tld groups
hosts = set( [ rec[dataprocess.HOST] for rec in wlcg_records ] )
tld_groups = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# create composite to-tier names
vo_tiers = set()
for rec in wlcg_records:
tl = 't1' if 'T1' in rec[dataprocess.TIER] else 't2'
vt = tl + '-' + rec[dataprocess.VO_NAME]
rec[dataprocess.VO_NAME] = vt
del rec[dataprocess.TIER] # same as collapsing afterwards
vo_tiers.add(vt)
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate total per site
site_totals = dataprocess.collapseFields(wlcg_records, ( dataprocess.VO_NAME, ) )
for r in site_totals:
r[dataprocess.VO_NAME] = TOTAL
# calculate total per country-tier
country_tier_totals = [ r.copy() for r in wlcg_records ]
for rec in country_tier_totals:
rec[dataprocess.HOST] = rec[dataprocess.HOST].split('.')[-1].upper() + '-TOTAL'
rec[dataprocess.USER] = 'FAKE'
country_tier_totals = dataprocess.collapseFields(country_tier_totals, ( dataprocess.USER, ) )
# calculate total per country
country_totals = dataprocess.collapseFields(country_tier_totals, ( dataprocess.VO_NAME, ) )
for rec in country_totals:
rec[dataprocess.VO_NAME] = TOTAL
# calculate total per tier-vo
tier_vo_totals = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, ) )
for r in tier_vo_totals:
r[dataprocess.HOST] = TIER_TOTAL
# calculate total
total = dataprocess.collapseFields(wlcg_records, ( dataprocess.HOST, dataprocess.VO_NAME ) )
assert len(total) in (0,1), 'Records did not collapse into a single record when calculating grand total'
if len(total) == 0:
total = [ { dataprocess.CPU_TIME : 0, dataprocess.WALL_TIME : 0, dataprocess.KSI2K_CPU_TIME : 0, dataprocess.KSI2K_WALL_TIME : 0 } ]
total_record = total[0]
total_record[dataprocess.HOST] = TIER_TOTAL
total_record[dataprocess.VO_NAME] = TOTAL
# put all calculated records together and add equivalents
wlcg_records += site_totals
wlcg_records += country_tier_totals
wlcg_records += country_totals
wlcg_records += tier_vo_totals
wlcg_records += [ total_record ]
wlcg_records = dataprocess.addEquivalentProperties(wlcg_records, days)
# create table
columns = sorted(vo_tiers)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += tld_groups[tld]
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
unit_extractor = WLCG_UNIT_MAPPING.get(unit, WLCG_UNIT_MAPPING_DEFAULT)
elements = []
for row in row_names:
for col in columns:
for rec in wlcg_records:
if rec[dataprocess.HOST] == row and rec[dataprocess.VO_NAME] == col:
value = _formatValue( unit_extractor(rec) )
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
# render page
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG oversight view'
unit_options = [ ( 'ksi2k-ne', 'KSI2K Node Equivalents (default)') , ( 'hs06-ne', 'HS06 Node Equivalents' ),
( 'ksi2k-wallhours', 'KSI2K Walltime Hours' ) , ( 'hs06-wallhours', 'HS06 Walltime hours') ]
unit_buttons = html.createRadioButtons('unit', unit_options, checked_value=unit)
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option, unit_buttons)
quarters = dateform.generateFormQuarters()
quarter_links = []
for q in quarters:
year, quart = dateform.parseQuarter(q)
sd, ed = dateform.quarterStartEndDates(year, quart)
quarter_links.append(html.createLink('%s?startdate=%s&enddate=%s' % (self.path, sd, ed), q ) )
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph('Quarters: \n ' + (' ' + html.NBSP).join(quarter_links) ) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderMachineView(self, results, request):
machine_manifest = results[0][1][0]
jobs_per_day = results[1][1]
top_projects = results[2][1]
top_users = results[3][1]
first_record_registration = machine_manifest[0]
last_record_registration = machine_manifest[1]
first_job_start = machine_manifest[2]
last_job_termination = machine_manifest[3]
distinct_users = machine_manifest[4]
distinct_projects = machine_manifest[5]
n_jobs = machine_manifest[6]
e_batches = [e[0] for e in jobs_per_day] # dates
e_groups = ['jobs']
e_matrix = dict([((e[0], e_groups[0]), e[1]) for e in jobs_per_day])
executed_table = htmltable.createHTMLTable(e_matrix,
e_batches,
e_groups,
base_indent=4)
stats_batches = ['Walltime days', 'Efficiency', 'Number of jobs']
p_groups = [e[0] for e in top_projects] # user list
p_dict_elements = []
p_dict_elements += [((stats_batches[0], p[0]), p[1])
for p in top_projects]
p_dict_elements += [((stats_batches[1], p[0]), p[2])
for p in top_projects]
p_dict_elements += [((stats_batches[2], p[0]), p[3])
for p in top_projects]
p_matrix = dict(p_dict_elements)
project_table = htmltable.createHTMLTable(p_matrix,
stats_batches,
p_groups,
base_indent=4)
u_groups = [e[0] for e in top_users] # user list
u_dict_elements = []
u_dict_elements += [((stats_batches[0], u[0]), u[1])
for u in top_users]
u_dict_elements += [((stats_batches[1], u[0]), u[2])
for u in top_users]
u_dict_elements += [((stats_batches[2], u[0]), u[3])
for u in top_users]
u_matrix = dict(u_dict_elements)
user_table = htmltable.createHTMLTable(u_matrix,
stats_batches,
u_groups,
base_indent=4)
title = 'Machine view for %s' % self.machine_name
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
# generate year-month options one year back
month_options = ['']
gmt = time.gmtime()
for i in range(gmt.tm_mon - 12, gmt.tm_mon + 1):
if i <= 0:
month_options.append('%i-%02d' % (gmt.tm_year - 1, 12 + i))
elif i > 0:
month_options.append('%i-%02d' % (gmt.tm_year, i))
sel1 = html.createSelector('Start month', 'startdate', month_options,
start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options,
end_date_option)
selector_form = html.createSelectorForm(self.machine_name,
[sel1, sel2])
if start_date_option or end_date_option:
range_text = 'selected date range'
else:
range_text = 'current month'
# create page
request.write(html.HTML_VIEWBASE_HEADER % {'title': title})
request.write(html.createTitle(title))
request.write('\n' + selector_form + '\n')
if not (start_date_option or end_date_option):
# skip manifest / inserts if date range is selected
request.write(html.createSectionTitle('Manifest'))
request.write(
html.createParagraph('First record registration: %s' %
first_record_registration))
request.write(
html.createParagraph('Last record registration: %s' %
last_record_registration))
request.write(
html.createParagraph('First job start: %s' % first_job_start))
request.write(
html.createParagraph('Last job termination: %s' %
last_job_termination))
request.write(
html.createParagraph('Distinct users: %s' % distinct_users))
request.write(
html.createParagraph('Distinct projects: %s' %
distinct_projects))
request.write(html.createParagraph('Number of jobs: %s' % n_jobs))
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Executed jobs in the last ten days'))
request.write(executed_table)
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Top 10 projects for the %s' % range_text))
request.write(project_table)
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Top 20 users for the %s' % range_text))
request.write(user_table)
request.write(html.P + '\n')
request.write(html.HTML_VIEWBASE_FOOTER)
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date,
unit, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
t_dataprocess_start = time.time()
wlcg_records = dataprocess.rowsToDicts(wlcg_data)
wlcg_records = [
r for r in wlcg_records
if r[dataprocess.VO_NAME] in ('alice', 'atlas')
]
# Separate the records into computing and storage
comp_records = []
storage_records = []
for r in wlcg_records:
if r[dataprocess.HOST] == 'STORAGE':
storage_records.append(r)
else:
comp_records.append(r)
# massage data
comp_records = dataprocess.addMissingScaleValues(
comp_records, self.hepspec06)
comp_records = dataprocess.collapseFields(comp_records, self.collapse)
comp_records = dataprocess.tierMergeSplit(comp_records,
self.tier_mapping,
self.tier_shares,
self.default_tier)
# Collapse the fields that couldn't be collapsed before the tier-splitting.
comp_records = [r for r in comp_records if r['tier'] == u'NDGF-T1']
comp_records = dataprocess.collapseFields(
comp_records, [dataprocess.VO_ROLE, dataprocess.HOST])
summary = {}
for r in comp_records + storage_records:
vo = r[dataprocess.VO_NAME]
if vo not in summary:
summary[vo] = {
'disk': 0.0,
'tape': 0.0,
dataprocess.HS06_WALL_TIME: 0.0,
dataprocess.HS06_CPU_TIME: 0.0
}
if r.get(dataprocess.HOST, '') == 'STORAGE':
media = r[
'vo_group'] # We stored "storage_media" in "vo_group" ...
summary[vo][media] += r[
dataprocess.
N_JOBS] # ... and "resource_capacity_used" in "n_jobs".
else:
summary[vo][dataprocess.HS06_WALL_TIME] += r[
dataprocess.HS06_WALL_TIME]
summary[vo][dataprocess.HS06_CPU_TIME] += r[
dataprocess.HS06_CPU_TIME]
columns = [("Walltime (HS06 days)",
lambda r: r[dataprocess.HS06_WALL_TIME] / 24.0),
("CPUtime (HS06 days)",
lambda r: r[dataprocess.HS06_CPU_TIME] / 24.0),
("Disk (TiB)", lambda r: r['disk'] / 1024.0**4),
("Tape (TiB)", lambda r: r['tape'] / 1024.0**4)]
elements = []
for row in summary.keys():
for cname, cfunc in columns:
value = "%.2f" % cfunc(summary[row])
elements.append(((cname, row), value))
t_dataprocess = time.time() - t_dataprocess_start
matrix = dict(elements)
table_content = htmltable.createHTMLTable(matrix,
[c[0] for c in columns],
summary.keys())
# render page
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG T1 Summary'
selector_form = dateform.createMonthSelectorForm(
self.path, start_date_option, end_date_option)
quarters = dateform.generateFormQuarters()
quarter_links = []
for q in quarters:
year, quart = dateform.parseQuarter(q)
sd, ed = dateform.quarterStartEndDates(year, quart)
quarter_links.append(
html.createLink(
'%s?startdate=%s&enddate=%s' % (self.path, sd, ed), q))
range_text = html.createParagraph('Date range: %s - %s (%s days)' %
(start_date, end_date, days))
request.write(html.HTML_VIEWBASE_HEADER % {'title': title})
request.write(html.createTitle(title))
request.write(
html.createParagraph('Quarters: \n ' +
(' ' + html.NBSP).join(quarter_links)))
request.write(html.SECTION_BREAK)
request.write(html.createParagraph(selector_form))
request.write(html.SECTION_BREAK)
request.write(html.createParagraph(range_text))
request.write(table_content)
request.write(html.SECTION_BREAK)
request.write(
html.createParagraph('Query time: %s' % round(t_query, 2)))
request.write(
html.createParagraph('Data process time: %s' %
round(t_dataprocess, 2)))
request.write(html.HTML_VIEWBASE_FOOTER)
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, wlcg_data, request, start_date, end_date, unit, t_query_start):
t_query = time.time() - t_query_start
days = dateform.dayDelta(start_date, end_date)
t_dataprocess_start = time.time()
wlcg_records = wlcg.rowsToDicts(wlcg_data, [ wlcg.MACHINE_NAME, wlcg.COUNTRY, wlcg.VO_NAME, unit ])
t_dataprocess = time.time() - t_dataprocess_start
# get tld groups
tld_groups = {}
for rec in wlcg_records:
host = rec[wlcg.MACHINE_NAME]
tld = _countryCode(rec)
if tld not in tld_groups:
tld_groups[tld] = [host]
elif host not in tld_groups[tld]:
tld_groups[tld].append(host)
vo_tiers = set()
for rec in wlcg_records:
vo_tiers.add(rec[wlcg.VO_NAME])
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
site_totals = _collapseFields(wlcg_records, ( wlcg.VO_NAME, ) )
for r in site_totals:
r[wlcg.VO_NAME] = TOTAL
country_vo_aggregate = {}
for r in wlcg_records:
machine = _countryCode(r) + '-TOTAL'
if machine not in country_vo_aggregate:
country_vo_aggregate[machine] = {}
vo = r[wlcg.VO_NAME]
if vo not in country_vo_aggregate[machine]:
country_vo_aggregate[machine][vo] = 0
country_vo_aggregate[machine][vo] += r[unit]
country_tier_totals = []
for machine in country_vo_aggregate:
for vo in country_vo_aggregate[machine]:
r = {wlcg.MACHINE_NAME: machine, wlcg.VO_NAME: vo, unit: country_vo_aggregate[machine][vo]}
country_tier_totals.append(r)
# calculate total per country
country_totals = _collapseFields(country_tier_totals, ( wlcg.VO_NAME, ) )
for rec in country_totals:
rec[wlcg.VO_NAME] = TOTAL
# calculate total per tier-vo
tier_vo_totals = _collapseFields(wlcg_records, ( wlcg.MACHINE_NAME, wlcg.COUNTRY ) )
for r in tier_vo_totals:
r[wlcg.MACHINE_NAME] = TIER_TOTAL
# calculate total
total = _collapseFields(wlcg_records, ( wlcg.MACHINE_NAME, wlcg.VO_NAME, wlcg.COUNTRY ) )
assert len(total) in (0,1), 'Records did not collapse into a single record when calculating grand total'
if len(total) == 0:
total = [ { wlcg.CPU_SECONDS : 0, wlcg.CORE_SECONDS : 0, wlcg.CPU_SECONDS_HS06 : 0, wlcg.CORE_SECONDS_HS06 : 0 } ]
total_record = total[0]
total_record[wlcg.MACHINE_NAME] = TIER_TOTAL
total_record[wlcg.VO_NAME] = TOTAL
# put all calculated records together and add equivalents
wlcg_records += site_totals
wlcg_records += country_tier_totals
wlcg_records += country_totals
wlcg_records += tier_vo_totals
wlcg_records += [ total_record ]
# create table
columns = sorted(vo_tiers)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += sorted(tld_groups[tld])
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
#unit_extractor = WLCG_UNIT_MAPPING.get(unit, WLCG_UNIT_MAPPING_DEFAULT)
unit_extractor = lambda rec : rec[unit]
elements = []
for row in row_names:
for col in columns:
for rec in wlcg_records:
if rec[wlcg.MACHINE_NAME] == row and rec[wlcg.VO_NAME] == col:
value = _formatValue( unit_extractor(rec) )
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
# render page
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
title = 'WLCG oversight view'
unit_options = []
for u in self.units:
unit_options.append(( u, COLUMN_NAMES[u] ))
unit_buttons = html.createRadioButtons('unit', unit_options, checked_value=unit)
selector_form = dateform.createMonthSelectorForm(self.path, start_date_option, end_date_option, unit_buttons)
quarters = dateform.generateFormQuarters()
quarter_links = []
for q in quarters:
year, quart = dateform.parseQuarter(q)
sd, ed = dateform.quarterStartEndDates(year, quart)
quarter_links.append(html.createLink('%s?startdate=%s&enddate=%s' % (self.path, sd, ed), q ) )
range_text = html.createParagraph('Date range: %s - %s (%s days)' % (start_date, end_date, days))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph('Quarters: \n ' + (' ' + html.NBSP).join(quarter_links) ) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(range_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET