def createMonthSelectorForm(baseurl, start_date_option, end_date_option, buttons=None):
month_options = generateMonthFormOptions()
sel1 = html.createSelector('Start month', 'startdate', month_options, start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options, end_date_option)
selector_form = html.createSelectorForm(baseurl, [ sel1, sel2 ], buttons)
return selector_form
def createMonthSelectorForm(baseurl,
start_date_option,
end_date_option,
buttons=None):
month_options = generateMonthFormOptions()
sel1 = html.createSelector('Start month', 'startdate', month_options,
start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options,
end_date_option)
selector_form = html.createSelectorForm(baseurl, [sel1, sel2], buttons)
return selector_form
def createQuarterSelectorForm(baseurl, quarter):
quarter_options = generateQuarterFormOptions()
sel = html.createSelector('Quarter', 'quarter', quarter_options, quarter)
selector_form = html.createSelectorForm(baseurl, [sel] )
return selector_form
def createQuarterSelectorForm(baseurl, quarter):
quarter_options = generateQuarterFormOptions()
sel = html.createSelector('Quarter', 'quarter', quarter_options, quarter)
selector_form = html.createSelectorForm(baseurl, [sel])
return selector_form
def renderWLCGViewPage(self, db_rows, request, date, media, t_query_start):
t_query = time.time() - t_query_start
t_dataprocess_start = time.time()
records = self.buildRecords(db_rows)
# remove infomration from certain groups, add they not add any value
records = [ rec for rec in records if rec['group'] not in ('dteam', 'behrmann') ]
if media == 'disk':
records = [ rec for rec in records if rec['media'] == 'disk' ]
elif media == 'tape':
records = [ rec for rec in records if rec['media'] == 'tape' ]
elif media == 'all':
pass
else:
return self.renderErrorPage('Invalid media selection', request)
# get top level domains and sites per country
hosts = set( [ rec['site'] for rec in records ] )
tld_groups = {}
country_sites = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# get groups
groups = set( [ rec['group'] for rec in records ] )
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate totals per site / group
site_group_totals = {}
for rec in records:
site, group, rcu = rec['site'], rec['group'], rec['rcu']
key = (site, group)
site_group_totals[key] = site_group_totals.get(key, 0) + rcu
# calculate total per site
site_totals = {}
for rec in records:
site, rcu = rec['site'], rec['rcu']
site_totals[site] = site_totals.get(site,0) + rcu
# calculate total per group / country
group_country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
key = (country, rec['group'])
group_country_totals[key] = group_country_totals.get(key, 0) + rec['rcu']
# calculate total per country
country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
country_totals[country] = country_totals.get(country,0) + rec['rcu']
# calculate total per group
group_totals = {}
for rec in records:
group_totals[rec['group']] = group_totals.get(rec['group'],0) + rec['rcu']
# calculate total
total = sum( [ rec['rcu'] for rec in records ] )
totals = []
# put all calculated records together and add equivalents
for (site, group), rcu in site_group_totals.items():
totals.append( { 'site': site, 'group': group, 'rcu': rcu } )
for site, rcu in site_totals.items():
totals.append( { 'site': site, 'group': TOTAL, 'rcu': rcu } )
for (country, group), rcu in group_country_totals.items():
totals.append( { 'site': country, 'group': group, 'rcu': rcu } )
for country, rcu in country_totals.items():
totals.append( { 'site': country, 'group': TOTAL, 'rcu': rcu } )
for group, rcu in group_totals.items():
totals.append( { 'site': TIER_TOTAL, 'group': group, 'rcu': rcu } )
totals.append( { 'site': TIER_TOTAL, 'group': TOTAL, 'rcu': total } )
# create table
columns = sorted(groups)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += sorted(tld_groups[tld])
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
elements = []
for row in row_names:
for col in columns:
for rec in totals:
if rec['site'] == row and rec['group'] == col:
value = rec['rcu']
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
t_dataprocess = time.time() - t_dataprocess_start
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
title = 'WLCG storage view'
media_options = [ ( 'all', 'Disk and Tape') , ( 'disk', 'Disk only' ), ( 'tape', 'Tape only' ) ]
media_buttons = html.createRadioButtons('media', media_options, checked_value=media)
month_options = dateform.generateMonthFormOptions()
selector = html.createSelector('Month', 'date', month_options, date)
selector_form = html.createSelectorForm(self.path, [ selector ], media_buttons )
date_text = html.createParagraph('Date: %s' % (date))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(date_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderMachineView(self, results, request):
machine_manifest = results[0][1][0]
jobs_per_day = results[1][1]
top_projects = results[2][1]
top_users = results[3][1]
first_record_registration = machine_manifest[0]
last_record_registration = machine_manifest[1]
first_job_start = machine_manifest[2]
last_job_termination = machine_manifest[3]
distinct_users = machine_manifest[4]
distinct_projects = machine_manifest[5]
n_jobs = machine_manifest[6]
e_batches = [ e[0] for e in jobs_per_day ] # dates
e_groups = [ 'jobs' ]
e_matrix = dict( [ ((e[0], e_groups[0]), e[1]) for e in jobs_per_day ] )
executed_table = htmltable.createHTMLTable(e_matrix, e_batches, e_groups, base_indent=4)
stats_batches = [ 'Walltime days', 'Efficiency', 'Number of jobs' ]
p_groups = [ e[0] for e in top_projects ] # user list
p_dict_elements = []
p_dict_elements += [ ((stats_batches[0], p[0]), p[1]) for p in top_projects ]
p_dict_elements += [ ((stats_batches[1], p[0]), p[2]) for p in top_projects ]
p_dict_elements += [ ((stats_batches[2], p[0]), p[3]) for p in top_projects ]
p_matrix = dict(p_dict_elements)
project_table = htmltable.createHTMLTable(p_matrix, stats_batches, p_groups, base_indent=4)
u_groups = [ e[0] for e in top_users ] # user list
u_dict_elements = []
u_dict_elements += [ ((stats_batches[0], u[0]), u[1]) for u in top_users ]
u_dict_elements += [ ((stats_batches[1], u[0]), u[2]) for u in top_users ]
u_dict_elements += [ ((stats_batches[2], u[0]), u[3]) for u in top_users ]
u_matrix = dict(u_dict_elements)
user_table = htmltable.createHTMLTable(u_matrix, stats_batches, u_groups, base_indent=4)
title = 'Machine view for %s' % self.machine_name
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
# generate year-month options one year back
month_options = ['']
gmt = time.gmtime()
for i in range(gmt.tm_mon-12, gmt.tm_mon+1):
if i <= 0:
month_options.append('%i-%02d' % (gmt.tm_year - 1, 12 + i) )
elif i > 0:
month_options.append('%i-%02d' % (gmt.tm_year, i) )
sel1 = html.createSelector('Start month', 'startdate', month_options, start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options, end_date_option)
selector_form = html.createSelectorForm(self.machine_name, [sel1, sel2] )
if start_date_option or end_date_option:
range_text = 'selected date range'
else:
range_text = 'current month'
# create page
request.write(html.HTML_VIEWBASE_HEADER % {'title': title})
request.write( html.createTitle(title) )
request.write('\n' + selector_form + '\n')
if not (start_date_option or end_date_option):
# skip manifest / inserts if date range is selected
request.write( html.createSectionTitle('Manifest') )
request.write( html.createParagraph('First record registration: %s' % first_record_registration) )
request.write( html.createParagraph('Last record registration: %s' % last_record_registration) )
request.write( html.createParagraph('First job start: %s' % first_job_start) )
request.write( html.createParagraph('Last job termination: %s' % last_job_termination) )
request.write( html.createParagraph('Distinct users: %s' % distinct_users) )
request.write( html.createParagraph('Distinct projects: %s' % distinct_projects) )
request.write( html.createParagraph('Number of jobs: %s' % n_jobs) )
request.write( html.SECTION_BREAK )
request.write( html.createSectionTitle('Executed jobs in the last ten days') )
request.write( executed_table )
request.write( html.SECTION_BREAK)
request.write( html.createSectionTitle('Top 10 projects for the %s' % range_text) )
request.write( project_table )
request.write( html.SECTION_BREAK )
request.write( html.createSectionTitle('Top 20 users for the %s' % range_text) )
request.write( user_table )
request.write( html.P + '\n' )
request.write(html.HTML_VIEWBASE_FOOTER)
request.finish()
return server.NOT_DONE_YET
def renderWLCGViewPage(self, db_rows, request, date, media, t_query_start):
t_query = time.time() - t_query_start
t_dataprocess_start = time.time()
records = self.buildRecords(db_rows)
# remove infomration from certain groups, add they not add any value
records = [ rec for rec in records if rec['group'] not in ('dteam', 'behrmann') ]
if media == 'disk':
records = [ rec for rec in records if rec['media'] == 'disk' ]
elif media == 'tape':
records = [ rec for rec in records if rec['media'] == 'tape' ]
elif media == 'all':
pass
else:
return self.renderErrorPage('Invalid media selection', request)
# get top level domains and sites per country
hosts = set( [ rec['site'] for rec in records ] )
tld_groups = {}
country_sites = {}
for host in hosts:
tld = host.split('.')[-1].upper()
tld_groups.setdefault(tld, []).append(host)
# get groups
groups = set( [ rec['group'] for rec in records ] )
TOTAL = 'Total'
TIER_TOTAL = self.default_tier.split('-')[0].upper()
# calculate totals per site / group
site_group_totals = {}
for rec in records:
site, group, rcu = rec['site'], rec['group'], rec['rcu']
key = (site, group)
site_group_totals[key] = site_group_totals.get(key, 0) + rcu
# calculate total per site
site_totals = {}
for rec in records:
site, rcu = rec['site'], rec['rcu']
site_totals[site] = site_totals.get(site,0) + rcu
# calculate total per group / country
group_country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
key = (country, rec['group'])
group_country_totals[key] = group_country_totals.get(key, 0) + rec['rcu']
# calculate total per country
country_totals = {}
for rec in records:
country = rec['site'].split('.')[-1].upper() + '-TOTAL'
country_totals[country] = country_totals.get(country,0) + rec['rcu']
# calculate total per group
group_totals = {}
for rec in records:
group_totals[rec['group']] = group_totals.get(rec['group'],0) + rec['rcu']
# calculate total
total = sum( [ rec['rcu'] for rec in records ] )
totals = []
# put all calculated records together and add equivalents
for (site, group), rcu in site_group_totals.items():
totals.append( { 'site': site, 'group': group, 'rcu': rcu } )
for site, rcu in site_totals.items():
totals.append( { 'site': site, 'group': TOTAL, 'rcu': rcu } )
for (country, group), rcu in group_country_totals.items():
totals.append( { 'site': country, 'group': group, 'rcu': rcu } )
for country, rcu in country_totals.items():
totals.append( { 'site': country, 'group': TOTAL, 'rcu': rcu } )
for group, rcu in group_totals.items():
totals.append( { 'site': TIER_TOTAL, 'group': group, 'rcu': rcu } )
totals.append( { 'site': TIER_TOTAL, 'group': TOTAL, 'rcu': total } )
# create table
columns = sorted(groups)
columns.append(TOTAL)
row_names = []
for tld in sorted(tld_groups):
row_names += sorted(tld_groups[tld])
row_names.append(tld + '-TOTAL')
row_names.append(TIER_TOTAL)
elements = []
for row in row_names:
for col in columns:
for rec in totals:
if rec['site'] == row and rec['group'] == col:
value = rec['rcu']
# hurrah for formatting
if row == TIER_TOTAL and col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True, double_underlined=True)
elif (row.endswith('-TOTAL') and col == TOTAL) or row == TIER_TOTAL:
value = htmltable.StyledTableValue(value, bold=True, underlined=True)
elif row.endswith('-TOTAL') or row == TIER_TOTAL or col == TOTAL:
value = htmltable.StyledTableValue(value, bold=True)
elements.append( ((col,row), value))
break
else:
elements.append( ((col,row), '') )
matrix = dict(elements)
t_dataprocess = time.time() - t_dataprocess_start
table_content = htmltable.createHTMLTable(matrix, columns, row_names, column_labels=COLUMN_NAMES)
title = 'WLCG storage view'
media_options = [ ( 'all', 'Disk and Tape') , ( 'disk', 'Disk only' ), ( 'tape', 'Tape only' ) ]
media_buttons = html.createRadioButtons('media', media_options, checked_value=media)
month_options = dateform.generateMonthFormOptions()
selector = html.createSelector('Month', 'date', month_options, date)
selector_form = html.createSelectorForm(self.path, [ selector ], media_buttons )
date_text = html.createParagraph('Date: %s' % (date))
request.write( html.HTML_VIEWBASE_HEADER % {'title': title} )
request.write( html.createTitle(title) )
request.write( html.createParagraph(selector_form) )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph(date_text) )
request.write( table_content )
request.write( html.SECTION_BREAK )
request.write( html.createParagraph('Query time: %s' % round(t_query, 2)) )
request.write( html.createParagraph('Data process time: %s' % round(t_dataprocess, 2)) )
request.write( html.HTML_VIEWBASE_FOOTER )
request.finish()
return server.NOT_DONE_YET
def renderMachineView(self, results, request):
machine_manifest = results[0][1][0]
jobs_per_day = results[1][1]
top_projects = results[2][1]
top_users = results[3][1]
first_record_registration = machine_manifest[0]
last_record_registration = machine_manifest[1]
first_job_start = machine_manifest[2]
last_job_termination = machine_manifest[3]
distinct_users = machine_manifest[4]
distinct_projects = machine_manifest[5]
n_jobs = machine_manifest[6]
e_batches = [e[0] for e in jobs_per_day] # dates
e_groups = ['jobs']
e_matrix = dict([((e[0], e_groups[0]), e[1]) for e in jobs_per_day])
executed_table = htmltable.createHTMLTable(e_matrix,
e_batches,
e_groups,
base_indent=4)
stats_batches = ['Walltime days', 'Efficiency', 'Number of jobs']
p_groups = [e[0] for e in top_projects] # user list
p_dict_elements = []
p_dict_elements += [((stats_batches[0], p[0]), p[1])
for p in top_projects]
p_dict_elements += [((stats_batches[1], p[0]), p[2])
for p in top_projects]
p_dict_elements += [((stats_batches[2], p[0]), p[3])
for p in top_projects]
p_matrix = dict(p_dict_elements)
project_table = htmltable.createHTMLTable(p_matrix,
stats_batches,
p_groups,
base_indent=4)
u_groups = [e[0] for e in top_users] # user list
u_dict_elements = []
u_dict_elements += [((stats_batches[0], u[0]), u[1])
for u in top_users]
u_dict_elements += [((stats_batches[1], u[0]), u[2])
for u in top_users]
u_dict_elements += [((stats_batches[2], u[0]), u[3])
for u in top_users]
u_matrix = dict(u_dict_elements)
user_table = htmltable.createHTMLTable(u_matrix,
stats_batches,
u_groups,
base_indent=4)
title = 'Machine view for %s' % self.machine_name
start_date_option = request.args.get('startdate', [''])[0]
end_date_option = request.args.get('enddate', [''])[0]
# generate year-month options one year back
month_options = ['']
gmt = time.gmtime()
for i in range(gmt.tm_mon - 12, gmt.tm_mon + 1):
if i <= 0:
month_options.append('%i-%02d' % (gmt.tm_year - 1, 12 + i))
elif i > 0:
month_options.append('%i-%02d' % (gmt.tm_year, i))
sel1 = html.createSelector('Start month', 'startdate', month_options,
start_date_option)
sel2 = html.createSelector('End month', 'enddate', month_options,
end_date_option)
selector_form = html.createSelectorForm(self.machine_name,
[sel1, sel2])
if start_date_option or end_date_option:
range_text = 'selected date range'
else:
range_text = 'current month'
# create page
request.write(html.HTML_VIEWBASE_HEADER % {'title': title})
request.write(html.createTitle(title))
request.write('\n' + selector_form + '\n')
if not (start_date_option or end_date_option):
# skip manifest / inserts if date range is selected
request.write(html.createSectionTitle('Manifest'))
request.write(
html.createParagraph('First record registration: %s' %
first_record_registration))
request.write(
html.createParagraph('Last record registration: %s' %
last_record_registration))
request.write(
html.createParagraph('First job start: %s' % first_job_start))
request.write(
html.createParagraph('Last job termination: %s' %
last_job_termination))
request.write(
html.createParagraph('Distinct users: %s' % distinct_users))
request.write(
html.createParagraph('Distinct projects: %s' %
distinct_projects))
request.write(html.createParagraph('Number of jobs: %s' % n_jobs))
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Executed jobs in the last ten days'))
request.write(executed_table)
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Top 10 projects for the %s' % range_text))
request.write(project_table)
request.write(html.SECTION_BREAK)
request.write(
html.createSectionTitle('Top 20 users for the %s' % range_text))
request.write(user_table)
request.write(html.P + '\n')
request.write(html.HTML_VIEWBASE_FOOTER)
request.finish()
return server.NOT_DONE_YET