def agentsSites(url):
"Return list of sites known in CMS WMAgents"
sites_ready_in_agent = set()
headers = {'Accept': 'application/json'}
params = {}
mgr = RequestHandler()
res = mgr.getdata(url,
params=params,
headers=headers,
ckey=ckey(),
cert=cert())
data = json.loads(res)
agents = {}
for r in [i['value'] for i in data['rows']]:
team = r['agent_team']
if team != 'production':
continue
agents.setdefault(team, []).append(r)
for team, agents in viewitems(agents):
for agent in agents:
if agent['status'] != 'ok':
continue
for site, sinfo in viewitems(agent['WMBS_INFO']['thresholds']):
if sinfo['state'] in ['Normal']:
sites_ready_in_agent.add(site)
return sites_ready_in_agent
def getMSOutputTransferInfo(self, wflow):
"""
Fetches the transfer information from the MSOutput REST interface for
the given workflow.
:param wflow: A MSRuleCleaner workflow representation
:return wflow:
"""
headers = {'Accept': 'application/json'}
params = {}
url = '%s/data/info?request=%s' % (self.msConfig['msOutputUrl'],
wflow['RequestName'])
try:
res = self.curlMgr.getdata(url,
params=params,
headers=headers,
ckey=ckey(),
cert=cert())
data = json.loads(res)['result'][0]
transferInfo = data['transferDoc']
except Exception as ex:
msg = "General exception while fetching TransferInfo from MSOutput for %s. "
msg += "Error: %s"
self.logger.exception(msg, wflow['RequestName'], str(ex))
if transferInfo is not None and transferInfo[
'TransferStatus'] == 'done':
wflow['TransferDone'] = True
return wflow
def getSpec(self, request, reqSpecs=None):
"Get request from workload cache"
if reqSpecs and request['RequestName'] in reqSpecs:
return reqSpecs[request['RequestName']]
url = str('%s/%s/spec' % (self.msConfig['reqmgrCacheUrl'], request['RequestName']))
mgr = RequestHandler()
data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey())
return pickle.loads(data)
def postRequest(self, params):
"Perform POST request to our MicroService"
headers = {'Content-type': 'application/json'}
print("### post call %s params=%s headers=%s" % (self.url, params, headers))
data = self.mgr.getdata(self.url, params=params, headers=headers, \
verb='POST', cert=cert(), ckey=ckey())
print("### post call data %s" % data)
return json.loads(data)
def getSpec(request, reqSpecs=None):
"Get request from workload cache"
if reqSpecs and request['RequestName'] in reqSpecs:
return reqSpecs[request['RequestName']]
url = str('%s/%s/spec' % (reqmgrCacheUrl(), request['RequestName']))
mgr = RequestHandler()
data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey())
return pickle.loads(data)
def findParent(dataset):
"Helper function to find a parent of the dataset"
url = '%s/datasetparents' % dbsUrl()
params = {'dataset': dataset}
headers = {'Accept': 'application/json'}
mgr = RequestHandler()
data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
return [str(i['parent_dataset']) for i in json.loads(data)]
def postRequest(self, apiName, params):
"Perform POST request to our MicroService"
headers = {'Content-type': 'application/json'}
url = self.url + "/%s" % apiName
data = self.mgr.getdata(url, params=params, headers=headers, \
verb='POST', cert=cert(), ckey=ckey(), encode=True, decode=True)
print("### post call data %s" % data)
return data
def getRequestSpecs(requestNames):
"Helper function to get all specs for given set of request names"
urls = [str('%s/%s/spec' % (reqmgrCacheUrl(), r)) for r in requestNames]
data = multi_getdata(urls, ckey(), cert())
rdict = {}
for row in data:
req = row['url'].split('/')[-2]
rdict[req] = pickle.loads(row['data'])
return rdict
def postRequest(self, params):
"Perform POST request to our MicroService"
headers = {'Content-type': 'application/json'}
print("### post call %s params=%s headers=%s" %
(self.url, params, headers))
data = self.mgr.getdata(self.url, params=params, headers=headers, \
verb='POST', cert=cert(), ckey=ckey())
print("### post call data %s" % data)
return json.loads(data)
def _getRequestSpecs(self, requestNames):
"Helper function to get all specs for given set of request names"
urls = [str('%s/%s/spec' % (self.msConfig['reqmgrCacheUrl'], r)) for r in requestNames]
data = multi_getdata(urls, ckey(), cert())
rdict = {}
for row in data:
req = row['url'].split('/')[-2]
rdict[req] = pickle.loads(row['data'])
return rdict
def getNodes(kind):
"Get list of PhEDEx nodes"
params = {}
headers = {'Accept': 'application/json'}
url = '%s/nodes' % phedexUrl()
mgr = RequestHandler()
data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
nodes = json.loads(data)['phedex']['node']
return [node['name'] for node in nodes if node['kind'] == kind]
def getNodes(kind):
"Get list of PhEDEx nodes"
params = {}
headers = {'Accept': 'application/json'}
url = '%s/nodes' % phedexUrl()
mgr = RequestHandler()
data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
nodes = json.loads(data)['phedex']['node']
return [node['name'] for node in nodes if node['kind'] == kind]
def findParent(dataset):
"Helper function to find a parent of the dataset"
url = '%s/datasetparents' % dbsUrl()
params = {'dataset': dataset}
headers = {'Accept': 'application/json'}
mgr = RequestHandler()
data = mgr.getdata(url,
params=params,
headers=headers,
cert=cert(),
ckey=ckey())
return [str(i['parent_dataset']) for i in json.loads(data)]
def getNodeQueues():
"Helper function to fetch nodes usage from PhEDEx data service"
headers = {'Accept': 'application/json'}
params = {}
mgr = RequestHandler()
url = '%s/nodeusagehistory' % phedexUrl()
res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
data = json.loads(res)
ret = defaultdict(int)
for node in data['phedex']['node']:
for usage in node['usage']:
ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB
return ret
def getNodeQueues():
"Helper function to fetch nodes usage from PhEDEx data service"
headers = {'Accept': 'application/json'}
params = {}
mgr = RequestHandler()
url = '%s/nodeusagehistory' % phedexUrl()
res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
data = json.loads(res)
ret = defaultdict(int)
for node in data['phedex']['node']:
for usage in node['usage']:
ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB
return ret
def workqueueRequests(state=None):
"Helper functions to get requests from WorkQueue"
url = workqueueView('jobsByRequest')
if state:
pass # we may need to use state when we'll query WorkQueue
params = {}
headers = {'Accept': 'application/json'}
mgr = RequestHandler()
data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
data = json.loads(data)
rdict = {}
for row in data.get('rows', []):
rdict[row['key']] = row['value']
return rdict
def getRequestWorkflows(requestNames):
"Helper function to get all specs for given set of request names"
urls = [str('%s/data/request/%s' % (reqmgrUrl(), r)) for r in requestNames]
data = multi_getdata(urls, ckey(), cert())
rdict = {}
for row in data:
req = row['url'].split('/')[-1]
try:
data = json.loads(row['data'])
rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict
except Exception as exp:
print("ERROR: fail to load data as json record, error=%s" % str(exp))
print(row)
return rdict
def getRequestWorkflows(requestNames):
"Helper function to get all specs for given set of request names"
urls = [str('%s/data/request/%s' % (reqmgrUrl(), r)) for r in requestNames]
data = multi_getdata(urls, ckey(), cert())
rdict = {}
for row in data:
req = row['url'].split('/')[-1]
try:
data = json.loads(row['data'])
rdict[req] = data['result'][
0] # we get back {'result': [workflow]} dict
except Exception as exp:
print("ERROR: fail to load data as json record, error=%s" %
str(exp))
print(row)
return rdict
def _getRequestWorkflows(self, requestNames):
"Helper function to get all specs for given set of request names"
urls = [str('%s/data/request/%s' % (self.msConfig['reqmgr2Url'], r)) for r in requestNames]
self.logger.debug("getRequestWorkflows")
for u in urls:
self.logger.debug("url %s", u)
data = multi_getdata(urls, ckey(), cert())
rdict = {}
for row in data:
req = row['url'].split('/')[-1]
try:
data = json.loads(row['data'])
rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict
except Exception as exp:
self.logger.error("fail to process row %s", row)
self.logger.exception("fail to load data as json record, error=%s", str(exp))
return rdict
def workqueueRequests(state=None):
"Helper functions to get requests from WorkQueue"
url = workqueueView('jobsByRequest')
if state:
pass # we may need to use state when we'll query WorkQueue
params = {}
headers = {'Accept': 'application/json'}
mgr = RequestHandler()
data = mgr.getdata(url,
params=params,
headers=headers,
cert=cert(),
ckey=ckey())
data = json.loads(data)
rdict = {}
for row in data.get('rows', []):
rdict[row['key']] = row['value']
return rdict
def agentsSites(url):
"Return list of sites known in CMS WMAgents"
sites_ready_in_agent = set()
headers = {'Accept': 'application/json'}
params = {}
mgr = RequestHandler()
res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
data = json.loads(res)
agents = {}
for r in [i['value'] for i in data['rows']]:
team = r['agent_team']
if team != 'production':
continue
agents.setdefault(team, []).append(r)
for team, agents in agents.items():
for agent in agents:
if agent['status'] != 'ok':
continue
for site, sinfo in agent['WMBS_INFO']['thresholds'].iteritems():
if sinfo['state'] in ['Normal']:
sites_ready_in_agent.add(site)
return sites_ready_in_agent
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
dashboardUrl = "http://dashb-ssb.cern.ch/dashboard/request.py"
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl,
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl,
### FIXME: these calls to gwmsmon are failing pretty badly with
### "302 Found" and failing to decode, causing a huge error dump
### to the logs
# 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/poolview/json/totals',
# 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'http://cmsmonitoring.web.cern.ch/cmsmonitoring/storageoverview/latest/StorageOverview.json',
]
cookie = {}
ssbids = [
'106', '107', '108', '109', '136', '158', '159', '160', '237'
]
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception as exc:
self.logger.exception('error %s for row %s', str(exc), row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
return siteInfo
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(),
### FIXME: these calls to gwmsmon are failing pretty badly with
### "302 Found" and failing to decode, causing a huge error dump
### to the logs
# 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
# 'https://cms-gwmsmon.cern.ch/poolview/json/totals',
# 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(),
]
cookie = {}
ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237']
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception:
traceback.print_exc()
print(row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl())
return siteInfo
def fetch(self):
"Fetch information about sites from various CMS data-services"
tfile = tempfile.NamedTemporaryFile()
urls = [
'%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(),
'%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(),
'https://cms-gwmsmon.cern.ch/totalview/json/site_summary',
'https://cms-gwmsmon.cern.ch/prodview/json/site_summary',
'https://cms-gwmsmon.cern.ch/poolview/json/totals',
'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus',
'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json',
'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt',
'%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(),
'%s/stuck_1.json' % stucktransferUrl(),
'%s/stuck_2.json' % stucktransferUrl(),
'%s/stuck_m1.json' % stucktransferUrl(),
'%s/stuck_m3.json' % stucktransferUrl(),
'%s/stuck_m4.json' % stucktransferUrl(),
'%s/stuck_m5.json' % stucktransferUrl(),
'%s/stuck_m6.json' % stucktransferUrl(),
]
cookie = {}
ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237']
sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6']
for url in urls:
if 'gwmsmon' in url:
cern_sso_cookie(url, tfile.name, cert(), ckey())
cookie.update({url: tfile.name})
gen = multi_getdata(urls, ckey(), cert(), cookie=cookie)
siteInfo = {}
for row in gen:
if 'Detox' in row['url']:
data = row['data']
else:
try:
data = json.loads(row['data'])
except Exception:
traceback.print_exc()
print(row)
data = {}
if 'ssb' in row['url']:
for ssbid in ssbids:
if ssbid in row['url']:
siteInfo['ssb_%s' % ssbid] = data
elif 'prodview/json/site_summary' in row['url']:
siteInfo['gwmsmon_prod_site_summary'] = data
elif 'totalview/json/site_summary' in row['url']:
siteInfo['gwmsmon_site_summary'] = data
elif 'totals' in row['url']:
siteInfo['gwmsmon_totals'] = data
elif 'maxusedcpus' in row['url']:
siteInfo['gwmsmon_prod_maxused'] = data
elif 'mcore' in row['url']:
siteInfo['mcore'] = data
elif 'Detox' in row['url']:
siteInfo['detox_sites'] = data
elif 'monitoring' in row['url']:
siteInfo['mss_usage'] = data
elif 'stuck' in row['url']:
for sid in sids:
if sid in row['url']:
siteInfo['stuck_%s' % sid] = data
siteInfo['site_queues'] = getNodeQueues()
siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl())
return siteInfo
