# morgue list
morgue = dashboard.parseMetric(url.read(sys.argv[2]))
# prepare hammercloud metric url for last 3 days!
hcURL = sys.argv[3] % (time.strftime("%Y-%m-%d", time.localtime(time.time()-3*24*60*60)),
time.strftime("%Y-%m-%d", time.localtime(time.time())))
hammerCloud = dashboard.parseJSONMetric(url.read(hcURL))
# get the url stamp for the dashboard input file
urlStamp = sys.argv[4]
# text output file location
txtOutput = sys.argv[5]
# json output file location
jsonOutput = sys.argv[6]
# create new metric object
metricHeader = {'twiki' : 'https://twiki.cern.ch/twiki/bin/view/CMSPublic/UsableSitesForAnalysis'}
metric = dashboard.metric(header = metricHeader)
def hasBadHistory(siteName):
# if the site is not in the hc metric, return False
# (you don't have any idea about the site, you cannot
# talk about it!)
if not siteName in hammerCloud.getSites():
return False
# if site has at least one green slot in the given time range
# (please check the hammerCloud link given as a parameter to see
# the time range), return False
slots = hammerCloud.getSiteEntries(siteName).values()
for slot in slots:
if slot.color == dashboard.green: return False
if term - canc - unkn < 10:
continue
# hammercloud value calculation
result = (succ - unsucc) / (term - canc - unkn) * 100.0
# check extreme cases
if (result < 0.0 or result > 100.0) or (term - canc - unkn < 0.0):
sys.stderr.write("something went really wrong! extreme case:\n")
sys.stderr.write(
"site:%s, app-succeeded:%s, unsuccess:%s, terminated:%s, allunk:%s, cancelled:%s"
% (name, succ, unsucc, term, unkn, canc)
)
continue
if name in hammerCloud:
hammerCloud[name] = result
production = dashboard.metric()
transitional = dashboard.metric()
for site in siteList:
badSiteFlag = False
errMsg = "bad"
siteDownTimeFlag = False
downTimeColors = ["grey", "yellow", "saddlebrown"]
# conditions to mark a site as bad
if samAccess[site] < 50.0 or samAccess[site] == "n/a":
badSiteFlag = badSiteFlag | True
if samAccess[site] == "n/a":
val = samAccess[site]
else:
val = round(samAccess[site], 2)
import sys
try: import json
except ImportError: import simplejson as json
from lib import fileOps, url, dashboard, sites
if len(sys.argv) < 3:
sys.stderr.write('not enough parameter!\n')
sys.exit(1)
federationSource = sys.argv[1]
metricOutput = sys.argv[2]
federations = json.loads(url.read(federationSource))
federationMetric = dashboard.metric()
for fedName in federations:
for site in federations[fedName]:
if fedName == 'prod':
color = dashboard.green
elif fedName == 'trans':
color = dashboard.cyan
elif fedName == 'nowhere':
color = dashboard.gray
else:
# basically, this is impossible state considering possible
# federation names but I wanted to consider this in case of
# a change. --and this change must be reflected to the metric.
color = dashboard.white
entry = dashboard.entry(None, site, fedName, color, federationSource)
federationMetric.append(entry)
parsed[cmsSite][id] = subject
# generate output for twiki meeting page
ticketURL = "https://ggus.eu/?mode=ticket_info&ticket_id="
twikiTable = "\n| *CMS Site* | *Number of Tickets* | * Tickets* |\n"
sum = 0
for site in parsed:
url = ""
sum = sum + len(parsed[site])
for id in parsed[site]:
url = url + "[[%s][%s]] " % (ticketURL + id, id)
twikiTable = twikiTable + "| %s | %d | %s |\n" % (site, len(
parsed[site]), url)
dateStamp = time.strftime("%d/%b/%Y %H:%M:%S (GMT)", time.gmtime())
twikiTable = twikiTable + "| *<i>generated on %s</i>, Total number of tickets: %s* |||" % (
dateStamp, sum)
fileOps.write(sys.argv[2], twikiTable)
# generate text file for the dashboard metric
metric = dashboard.metric()
allSites = sites.getSites().keys()
url = "https://ggus.eu/?mode=ticket_search&cms_site=%s&timeframe=any&status=open&search_submit=GO%%21"
for site in parsed:
value = len(parsed[site])
metric.append(dashboard.entry(None, site, value, dashboard.red,
url % site))
for site in allSites:
if site in parsed.keys(): continue
metric.append(dashboard.entry(None, site, 0, dashboard.green, url % site))
fileOps.write(sys.argv[3], str(metric))
if not cmsSite: continue
if not parsed.has_key(cmsSite):
parsed[cmsSite] = {}
parsed[cmsSite][id] = subject
# generate output for twiki meeting page
ticketURL = "https://ggus.eu/?mode=ticket_info&ticket_id="
twikiTable = "\n| *CMS Site* | *Number of Tickets* | * Tickets* |\n"
sum = 0
for site in parsed:
url = ""
sum = sum + len(parsed[site])
for id in parsed[site]:
url = url + "[[%s][%s]] " % (ticketURL + id, id)
twikiTable = twikiTable + "| %s | %d | %s |\n" % (site, len(parsed[site]), url)
dateStamp = time.strftime("%d/%b/%Y %H:%M:%S (GMT)", time.gmtime())
twikiTable = twikiTable + "| *<i>generated on %s</i>, Total number of tickets: %s* |||" % (dateStamp, sum)
fileOps.write(sys.argv[2], twikiTable)
# generate text file for the dashboard metric
metric = dashboard.metric()
allSites = sites.getSites().keys()
url = "https://ggus.eu/?mode=ticket_search&cms_site=%s&timeframe=any&status=open&search_submit=GO%%21"
for site in parsed:
value = len(parsed[site])
metric.append(dashboard.entry(None, site, value, dashboard.red, url % site))
for site in allSites:
if site in parsed.keys(): continue
metric.append(dashboard.entry(None, site, 0, dashboard.green, url % site))
fileOps.write(sys.argv[3], str(metric))
# a closed loop for the metric. when someone changes a value in the
# 'usable sites - manual changes' metric by using dashboard web interface,
# the script reflects this change to the input text file of the metric.
if len(sys.argv) < 3:
print 'not enough parameter!'
sys.exit(1)
# output path
output = sys.argv[2]
# get the source metric url
metricURL = sys.argv[1]
# get the entries of the metric
metric = dashboard.parseJSONMetric(url.read(metricURL))
updatedMetric = dashboard.metric()
# merege sites from the vo-feed and manual control meteric.
siteList = sites.getSites()
for site in metric.getSites():
if not site in siteList:
siteList[site] = {}
for i in siteList:
# if the site is not in the list add it (this is the
# case that will be happaned when they create new site
# in the site db)
if not metric.hasSite(i):
updatedMetric.append(
dashboard.entry(None, i, 'ready', dashboard.green, metricURL))
else:
unkn = float(i.find('allunk').text)
canc = float(i.find('cancelled').text)
name = i.find('name').text
## Andrea's approach
# check if enough number of job existing to calculate the hc value
if term - canc - unkn < 10: continue
# hammercloud value calculation
result = (succ - unsucc) / (term - canc - unkn) * 100.;
# check extreme cases
if (result < 0.0 or result > 100.0) or (term - canc - unkn < 0.0):
sys.stderr.write("something went really wrong! extreme case:\n")
sys.stderr.write("site:%s, app-succeeded:%s, unsuccess:%s, terminated:%s, allunk:%s, cancelled:%s" % (name, succ, unsucc, term, unkn, canc))
continue
if name in hammerCloud: hammerCloud[name] = result
production = dashboard.metric()
transitional = dashboard.metric()
for site in siteList:
badSiteFlag = False
errMsg = 'bad'
siteDownTimeFlag = False
downTimeColors = ['grey', 'yellow', 'saddlebrown']
# conditions to mark a site as bad
if samAccess[site] < 50.0 or samAccess[site] == 'n/a':
badSiteFlag = badSiteFlag | True
if samAccess[site] == 'n/a': val = samAccess[site]
else: val = round(samAccess[site], 2)
errMsg = errMsg + '_SAM(%s)' % val
if (hammerCloud[site] < 70.0 or hammerCloud[site] == 'n/a') and sites.getTier(site) != 3:
# this script provides data for the 'usable sites - manual changes' metric,
# which is created to control the 'usable sites' metric by hand, and creates
# a closed loop for the metric. when someone changes a value in the
# 'usable sites - manual changes' metric by using dashboard web interface,
# the script reflects this change to the input text file of the metric.
if len(sys.argv) < 3:
print 'not enough parameter!'
sys.exit(1)
# output path
output = sys.argv[2]
# get the source metric url
metricURL = sys.argv[1]
# get the entries of the metric
metric = dashboard.parseJSONMetric(url.read(metricURL))
updatedMetric = dashboard.metric()
for i in sites.getSites():
# if the site is not in the list add it (this is the
# case that will be happaned when they create new site
# in the site db)
if not metric.hasSite(i):
updatedMetric.append(dashboard.entry(None, i, 'ready', dashboard.green, metricURL))
else:
latestEntry = metric.getLatestEntry(i)
updatedMetric.append(dashboard.entry(None, i, latestEntry.value, latestEntry.color, metricURL))
fileOps.write(output, str(updatedMetric))
