def getCredentials(jsonKey):
jsonData = json.loads(fileOps.read(jsonKey))
privateKey = jsonData['private_key']
clientEmail = jsonData['client_email']
scope = ['https://www.googleapis.com/auth/calendar', 'https://www.googleapis.com/auth/calendar.readonly']
credentials = SignedJwtAssertionCredentials(clientEmail, privateKey, scope)
return credentials
from lib import url, fileOps
import sys, os, time, copy
try:
import json
from json import encoder
encoder.FLOAT_REPR = lambda o: format(o, '.2f')
except ImportError: import simplejson as json
if len(sys.argv) < 2:
sys.exit('not enough arguments!')
tests = ['sam', 'hc', 'ggus']
# python report.py $TMP/report.json $TMP/report_2weeks.json
# set variables given from outside
currentReport = json.loads(fileOps.read(sys.argv[1]))
htmlTemplate = fileOps.read(sys.argv[2])
reportFile = sys.argv[3]
htmlOutputPath = sys.argv[4]
if os.path.isfile(reportFile):
print 'loading 2 weeks report...'
report = json.loads(fileOps.read(reportFile))
else:
print 'creating new 2 weeks report...'
report = {'data' : {}}
# insert new test results
data = currentReport['data']
for site in data:
if not report['data'].has_key(site):
report['data'][site] = {}
try:
import json
except ImportError:
import simplejson as json
try:
import xml.etree.ElementTree as ET
except ImportError:
from elementtree import ElementTree as ET
if len(sys.argv) < 6:
sys.stderr.write("not enough parameter!\n")
sys.exit(1)
siteList = sites.getSites()
ggusXMLFile = fileOps.read(sys.argv[1])
ggus = {}
samAccessURL = sys.argv[2]
samAccess = {}
hcURL = sys.argv[3]
hammerCloud = {}
downTimesURL = sys.argv[4]
downTimes = dashboard.parseJSONMetric(url.read(downTimesURL))
siteDownTimes = {}
federations = json.loads(url.read(sys.argv[5]))
reportFile = sys.argv[6]
reportURL = sys.argv[7]
output = sys.argv[8]
report = {}
#!/usr/bin/python # aaltunda - [email protected] import sys, time from lib import fileOps, dashboard, sites try: import xml.etree.ElementTree as ET except ImportError: from elementtree import ElementTree as ET if len(sys.argv) < 4: print 'not enough parameters' sys.exit(1) tickets = fileOps.read(sys.argv[1]) tickets = ET.fromstring(tickets) parsed = {} for ticket in tickets.findall('ticket'): cmsSite = ticket.find('cms_site').text subject = ticket.find('subject').text id = ticket.find('request_id').text if not cmsSite: continue if not parsed.has_key(cmsSite): parsed[cmsSite] = {} parsed[cmsSite][id] = subject # generate output for twiki meeting page ticketURL = "https://ggus.eu/?mode=ticket_info&ticket_id=" twikiTable = "\n| *CMS Site* | *Number of Tickets* | * Tickets* |\n"
import sys, os, time
try: import json
except ImportError: import simplejson as json
from lib import fileOps
if len(sys.argv) < 4:
sys.stderr.write("not enough prameter!\n")
sys.exit(1)
htmlTemplate = fileOps.read(sys.argv[1])
info = sys.argv[2]
inOutPath = sys.argv[3]
data = {}
for i in os.listdir(inOutPath):
if not '.json' in i: continue
date = time.strftime("%B %d, %Y %H:%M:%S", time.localtime(int(i.replace('.json', ''))))
data[date] = json.loads(fileOps.read("{0}/{1}".format(inOutPath, i)))
jsData = {}
for date in data:
for site in data[date]:
if not site in jsData:
jsData[site] = {}
if data[date][site] == 'n/a': data[date][site] = -1.0
jsData[site][date] = data[date][site]
def allTheSame(data):
oldVal = None
val = None
for i in data:
#!/usr/bin/python # aaltunda - [email protected] import sys, time from lib import fileOps, dashboard, sites try: import xml.etree.ElementTree as ET except ImportError: from elementtree import ElementTree as ET if len(sys.argv) < 4: print 'not enough parameters' sys.exit(1) tickets = fileOps.read(sys.argv[1]) tickets = ET.fromstring(tickets) parsed = {} for ticket in tickets.findall('ticket'): cmsSite = ticket.find('cms_site').text subject = ticket.find('subject').text id = ticket.find('request_id').text if not cmsSite: continue if not parsed.has_key(cmsSite): parsed[cmsSite] = {} parsed[cmsSite][id] = subject # generate output for twiki meeting page ticketURL = "https://ggus.eu/?mode=ticket_info&ticket_id=" twikiTable = "\n| *CMS Site* | *Number of Tickets* | * Tickets* |\n" sum = 0 for site in parsed:
#- SAM access test < 50% for two weeks
#- HammerCloud[1] test success rete < 70% for two weeks
import sys, time, urllib
from lib import fileOps, url, dashboard, sites
try: import json
except ImportError: import simplejson as json
try: import xml.etree.ElementTree as ET
except ImportError: from elementtree import ElementTree as ET
if len(sys.argv) < 6:
sys.stderr.write('not enough parameter!\n')
sys.exit(1)
siteList = sites.getSites()
ggusXMLFile = fileOps.read(sys.argv[1])
ggus = {}
samAccessURL = sys.argv[2]
samAccess = {}
hcURL = sys.argv[3]
hammerCloud = {}
downTimesURL = sys.argv[4]
downTimes = dashboard.parseJSONMetric(url.read(downTimesURL))
siteDownTimes = {}
federations = json.loads(url.read(sys.argv[5]))
reportFile = sys.argv[6]
reportURL = sys.argv[7]
output = sys.argv[8]
report = {}
def check_federation_history(site_name):
# altunda - [email protected] import sys from lib import fileOps try: import json except ImportError: import simplejson as json # input: release list, release, new documentetion status if len(sys.argv) < 4: print >> sys.stderr, 'Error: not enough parameters.' sys.exit(1) # read release list to find one to document. relList = json.loads(fileOps.read(sys.argv[1])) print "## documentation status will be updated for %s, %s -> %s" % (sys.argv[2], relList[sys.argv[2]]['status'], sys.argv[3]) # update status relList[sys.argv[2]]['status'] = sys.argv[3] # write the updated file fileOps.write(sys.argv[1], json.dumps(relList, indent=2))
try:
import json
except ImportError:
import simplejson as json
try:
import xml.etree.ElementTree as ET
except ImportError:
from elementtree import ElementTree as ET
# input: config file, input/output file
if len(sys.argv) < 3:
print >> sys.stderr, 'Error: not enough parameters.'
sys.exit(1)
# read the config file and parse it
conf = json.loads(fileOps.read(sys.argv[1]))
# get list of announced cmssw releases and parse it
cmssw = ET.fromstring(url.read(conf['urlRelList']))
# get list of documented cmssw releases and parse it
docCMSSW = ET.fromstring(url.read(conf['urlDocList']))
def isDocNeeded(relName):
if re.match(conf['pattern'], relName): return True
else: return False
# cmssw XML structure
class CMSSW:
def __init__(self, name=None, arch=None, type=None, state=None):
import sys, time
from lib import dashboard, url, fileOps, sites
try: import json
except ImportError: import simplejson as json
if len(sys.argv) < 4:
sys.stderr.write('not enough parameter!\n')
sys.exit(1)
siteList = sites.getSites()
config = json.loads(fileOps.read(sys.argv[1]))
fieldList = config.keys()
fieldList.sort()
data = {}
# add field names
data['fields'] = fieldList
# load all fields from dashboard
fields = {}
for field in fieldList:
fields[field] = dashboard.parseMetric(url.read(config[field]))
print field, 'done...'
for site in siteList:
data[site] = []
for field in fieldList:
if not fields[field].hasSite(site):
data[site].append('black')
continue
data[site].append(fields[field].getSiteEntry(site).color)
parser = OptionParser()
parser.add_option("-k", "--key", dest="jsonKeyFile", help="Key FILE for google calendar API", metavar="FILE")
parser.add_option("-m", "--map", dest="mapFile", help="Map FILE to matche tiers and calendars", metavar="FILE")
parser.add_option("-s", "--source", dest="source", help="Calendar source url", metavar="URL")
parser.add_option("-r", "--range", dest="range", help="Time RANGE (window) in hours", type="int")
(options, args) = parser.parse_args()
if not options.jsonKeyFile or not options.mapFile or not options.source or not options.range:
sys.stderr.write('not enough argument. please run python main.py --help.\n')
sys.exit(1)
# create google calendar api
api = google.calendarAPI(options.jsonKeyFile)
# parse calendar-CMS site tier map file
map = json.loads(fileOps.read(options.mapFile))
# calculate lower bound parameter for google calendar api
lowerBound = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - options.range*60*60))
# get downtimes from dashboard and parse json
print 'get downtime entries from dashboard: %s' % options.source
dashboardDT = dashboard.getDowntimes(options.source, lowerBound)
# get all calendar under the google account
calendars = api.getCalendars()['items']
# delete all old events to flush the downtime calendar
for calendar in calendars:
# if curret calendar is not important for us, skip it
# !!! be careful, you dont want to delete events placed
# in other calendars. just delete events placed in related
# calendars and time range (window) !!!
if not calendar['summary'] in map: continue
import sys, os, time
try:
import json
except ImportError:
import simplejson as json
from lib import fileOps
if len(sys.argv) < 4:
sys.stderr.write("not enough prameter!\n")
sys.exit(1)
htmlTemplate = fileOps.read(sys.argv[1])
info = sys.argv[2]
inOutPath = sys.argv[3]
data = {}
for i in os.listdir(inOutPath):
if not '.json' in i: continue
date = time.strftime("%B %d, %Y %H:%M:%S",
time.localtime(int(i.replace('.json', ''))))
data[date] = json.loads(fileOps.read("{0}/{1}".format(inOutPath, i)))
jsData = {}
for date in data:
for site in data[date]:
if not site in jsData:
jsData[site] = {}
if data[date][site] == 'n/a': data[date][site] = -1.0
jsData[site][date] = data[date][site]
from lib import url, fileOps
import sys, os, time, copy
try:
import json
from json import encoder
encoder.FLOAT_REPR = lambda o: format(o, '.2f')
except ImportError:
import simplejson as json
if len(sys.argv) < 2:
sys.exit('not enough arguments!')
tests = ['sam', 'hc', 'ggus']
# python report.py $TMP/report.json $TMP/report_2weeks.json
# set variables given from outside
currentReport = json.loads(fileOps.read(sys.argv[1]))
htmlTemplate = fileOps.read(sys.argv[2])
reportFile = sys.argv[3]
htmlOutputPath = sys.argv[4]
if os.path.isfile(reportFile):
print 'loading 2 weeks report...'
report = json.loads(fileOps.read(reportFile))
else:
print 'creating new 2 weeks report...'
report = {'data': {}}
# insert new test results
data = currentReport['data']
for site in data:
if not report['data'].has_key(site):
# this script finds an available release to generate ref man.
# please note that the output of the script should be printed
# as one line to be parsed by the sheel script.
import sys
from lib import fileOps
try: import json
except ImportError: import simplejson as json
# input: releases list, label to find
if len(sys.argv) < 3:
print >> sys.stderr, 'Error: not enough parameters.'
sys.exit(1)
# read release list to find one to document. printFlag must be false,
# otherwise the parser in the shell script will stop execution.
rels2Doc = json.loads(fileOps.read(sys.argv[1], printFlag = False))
# get keys to sort them
keys = rels2Doc.keys()
# reverse order to catch newer one (first in first out huh?)
keys.sort(reverse = True)
for i in keys:
if rels2Doc[i]['status'] == sys.argv[2]:
print i, rels2Doc[i]['arch']
sys.exit(0)
# no release to document
print "-"
# this script generates list of undocumented CMSSW releases.
import os, sys, re
from lib import url, fileOps
try: import json
except ImportError: import simplejson as json
try: import xml.etree.ElementTree as ET
except ImportError: from elementtree import ElementTree as ET
# input: config file, input/output file
if len(sys.argv) < 3:
print >> sys.stderr, 'Error: not enough parameters.'
sys.exit(1)
# read the config file and parse it
conf = json.loads(fileOps.read(sys.argv[1]))
# get list of announced cmssw releases and parse it
cmssw = ET.fromstring(url.read(conf['urlRelList']))
# get list of documented cmssw releases and parse it
docCMSSW = ET.fromstring(url.read(conf['urlDocList']))
def isDocNeeded(relName):
if re.match(conf['pattern'], relName): return True
else: return False
# cmssw XML structure
class CMSSW:
def __init__(self, name = None, arch = None, type = None,
state = None):
self.name = name.strip()
