def testSingle(self):
"""
Test single call to CERN SSO url.
"""
# test RequestHandler
url = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
params = {}
tfile = tempfile.NamedTemporaryFile()
cern_sso_cookie(url, tfile.name, self.cert, self.ckey)
cookie = {url: tfile.name}
header, _ = self.mgr.request(url, params, cookie=cookie)
self.assertTrue(header.status, 200)
def testSingle(self):
"""
Test single call to CERN SSO url.
"""
# test RequestHandler
url = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
params = {}
tfile = tempfile.NamedTemporaryFile()
cern_sso_cookie(url, tfile.name, self.cert, self.ckey)
cookie = {url: tfile.name}
header, _ = self.mgr.request(url, params, cookie=cookie)
self.assertTrue(header.status, 200)
def testMulti(self):
"""
Test fetch of several urls at once, one of the url relies on CERN SSO.
"""
tfile = tempfile.NamedTemporaryFile()
url1 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/help"
url2 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/datatiers"
url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
cern_sso_cookie(url3, tfile.name, self.cert, self.ckey)
cookie = {url3: tfile.name}
urls = [url1, url2, url3]
data = getdata(urls, self.ckey, self.cert, cookie=cookie)
headers = 0
for row in data:
if '200 OK' in row['headers']:
headers += 1
self.assertTrue(headers, 3)
def testMulti(self):
"""
Test fetch of several urls at once, one of the url relies on CERN SSO.
"""
tfile = tempfile.NamedTemporaryFile()
url1 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/help"
url2 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/datatiers"
url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary"
cern_sso_cookie(url3, tfile.name, self.cert, self.ckey)
cookie = {url3: tfile.name}
urls = [url1, url2, url3]
data = getdata(urls, self.ckey, self.cert, cookie=cookie)
headers = 0
for row in data:
if '200 OK' in row['headers']:
headers += 1
self.assertTrue(headers, 3)
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(),
### FIXME: these calls to gwmsmon are failing pretty badly with
### "302 Found" and failing to decode, causing a huge error dump
### to the logs
# 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/poolview/json/totals',
# 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(),
]
cookie = {}
ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237']
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception:
traceback.print_exc()
print(row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl())
return siteInfo
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
dashboardUrl = "http://dashb-ssb.cern.ch/dashboard/request.py"
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl,
### FIXME: these calls to gwmsmon are failing pretty badly with
### "302 Found" and failing to decode, causing a huge error dump
### to the logs
# 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/poolview/json/totals',
# 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'http://cmsmonitoring.web.cern.ch/cmsmonitoring/storageoverview/latest/StorageOverview.json',
]
cookie = {}
ssbids = [
'106', '107', '108', '109', '136', '158', '159', '160', '237'
]
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception as exc:
self.logger.exception('error %s for row %s', str(exc), row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
return siteInfo
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(),
'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
'https://cms-gwmsmon.cern.ch/poolview/json/totals',
'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(),
'%s/stuck_1.json' % stucktransferUrl(),
'%s/stuck_2.json' % stucktransferUrl(),
'%s/stuck_m1.json' % stucktransferUrl(),
'%s/stuck_m3.json' % stucktransferUrl(),
'%s/stuck_m4.json' % stucktransferUrl(),
'%s/stuck_m5.json' % stucktransferUrl(),
'%s/stuck_m6.json' % stucktransferUrl(),
]
cookie = {}
ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237']
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception:
traceback.print_exc()
print(row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl())
return siteInfo
