def agentsSites(url): "Return list of sites known in CMS WMAgents" sites_ready_in_agent = set() headers = {'Accept': 'application/json'} params = {} mgr = RequestHandler() res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) data = json.loads(res) agents = {} for r in [i['value'] for i in data['rows']]: team = r['agent_team'] if team != 'production': continue agents.setdefault(team, []).append(r) for team, agents in viewitems(agents): for agent in agents: if agent['status'] != 'ok': continue for site, sinfo in viewitems(agent['WMBS_INFO']['thresholds']): if sinfo['state'] in ['Normal']: sites_ready_in_agent.add(site) return sites_ready_in_agent
def getMSOutputTransferInfo(self, wflow): """ Fetches the transfer information from the MSOutput REST interface for the given workflow. :param wflow: A MSRuleCleaner workflow representation :return wflow: """ headers = {'Accept': 'application/json'} params = {} url = '%s/data/info?request=%s' % (self.msConfig['msOutputUrl'], wflow['RequestName']) try: res = self.curlMgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) data = json.loads(res)['result'][0] transferInfo = data['transferDoc'] except Exception as ex: msg = "General exception while fetching TransferInfo from MSOutput for %s. " msg += "Error: %s" self.logger.exception(msg, wflow['RequestName'], str(ex)) if transferInfo is not None and transferInfo[ 'TransferStatus'] == 'done': wflow['TransferDone'] = True return wflow
def getSpec(self, request, reqSpecs=None): "Get request from workload cache" if reqSpecs and request['RequestName'] in reqSpecs: return reqSpecs[request['RequestName']] url = str('%s/%s/spec' % (self.msConfig['reqmgrCacheUrl'], request['RequestName'])) mgr = RequestHandler() data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey()) return pickle.loads(data)
def postRequest(self, params): "Perform POST request to our MicroService" headers = {'Content-type': 'application/json'} print("### post call %s params=%s headers=%s" % (self.url, params, headers)) data = self.mgr.getdata(self.url, params=params, headers=headers, \ verb='POST', cert=cert(), ckey=ckey()) print("### post call data %s" % data) return json.loads(data)
def getSpec(request, reqSpecs=None): "Get request from workload cache" if reqSpecs and request['RequestName'] in reqSpecs: return reqSpecs[request['RequestName']] url = str('%s/%s/spec' % (reqmgrCacheUrl(), request['RequestName'])) mgr = RequestHandler() data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey()) return pickle.loads(data)
def findParent(dataset): "Helper function to find a parent of the dataset" url = '%s/datasetparents' % dbsUrl() params = {'dataset': dataset} headers = {'Accept': 'application/json'} mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey()) return [str(i['parent_dataset']) for i in json.loads(data)]
def postRequest(self, apiName, params): "Perform POST request to our MicroService" headers = {'Content-type': 'application/json'} url = self.url + "/%s" % apiName data = self.mgr.getdata(url, params=params, headers=headers, \ verb='POST', cert=cert(), ckey=ckey(), encode=True, decode=True) print("### post call data %s" % data) return data
def getRequestSpecs(requestNames): "Helper function to get all specs for given set of request names" urls = [str('%s/%s/spec' % (reqmgrCacheUrl(), r)) for r in requestNames] data = multi_getdata(urls, ckey(), cert()) rdict = {} for row in data: req = row['url'].split('/')[-2] rdict[req] = pickle.loads(row['data']) return rdict
def postRequest(self, params): "Perform POST request to our MicroService" headers = {'Content-type': 'application/json'} print("### post call %s params=%s headers=%s" % (self.url, params, headers)) data = self.mgr.getdata(self.url, params=params, headers=headers, \ verb='POST', cert=cert(), ckey=ckey()) print("### post call data %s" % data) return json.loads(data)
def _getRequestSpecs(self, requestNames): "Helper function to get all specs for given set of request names" urls = [str('%s/%s/spec' % (self.msConfig['reqmgrCacheUrl'], r)) for r in requestNames] data = multi_getdata(urls, ckey(), cert()) rdict = {} for row in data: req = row['url'].split('/')[-2] rdict[req] = pickle.loads(row['data']) return rdict
def getNodes(kind): "Get list of PhEDEx nodes" params = {} headers = {'Accept': 'application/json'} url = '%s/nodes' % phedexUrl() mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) nodes = json.loads(data)['phedex']['node'] return [node['name'] for node in nodes if node['kind'] == kind]
def getNodes(kind): "Get list of PhEDEx nodes" params = {} headers = {'Accept': 'application/json'} url = '%s/nodes' % phedexUrl() mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) nodes = json.loads(data)['phedex']['node'] return [node['name'] for node in nodes if node['kind'] == kind]
def findParent(dataset): "Helper function to find a parent of the dataset" url = '%s/datasetparents' % dbsUrl() params = {'dataset': dataset} headers = {'Accept': 'application/json'} mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey()) return [str(i['parent_dataset']) for i in json.loads(data)]
def getNodeQueues(): "Helper function to fetch nodes usage from PhEDEx data service" headers = {'Accept': 'application/json'} params = {} mgr = RequestHandler() url = '%s/nodeusagehistory' % phedexUrl() res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) data = json.loads(res) ret = defaultdict(int) for node in data['phedex']['node']: for usage in node['usage']: ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB return ret
def getNodeQueues(): "Helper function to fetch nodes usage from PhEDEx data service" headers = {'Accept': 'application/json'} params = {} mgr = RequestHandler() url = '%s/nodeusagehistory' % phedexUrl() res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) data = json.loads(res) ret = defaultdict(int) for node in data['phedex']['node']: for usage in node['usage']: ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB return ret
def workqueueRequests(state=None): "Helper functions to get requests from WorkQueue" url = workqueueView('jobsByRequest') if state: pass # we may need to use state when we'll query WorkQueue params = {} headers = {'Accept': 'application/json'} mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey()) data = json.loads(data) rdict = {} for row in data.get('rows', []): rdict[row['key']] = row['value'] return rdict
def getRequestWorkflows(requestNames): "Helper function to get all specs for given set of request names" urls = [str('%s/data/request/%s' % (reqmgrUrl(), r)) for r in requestNames] data = multi_getdata(urls, ckey(), cert()) rdict = {} for row in data: req = row['url'].split('/')[-1] try: data = json.loads(row['data']) rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict except Exception as exp: print("ERROR: fail to load data as json record, error=%s" % str(exp)) print(row) return rdict
def getRequestWorkflows(requestNames): "Helper function to get all specs for given set of request names" urls = [str('%s/data/request/%s' % (reqmgrUrl(), r)) for r in requestNames] data = multi_getdata(urls, ckey(), cert()) rdict = {} for row in data: req = row['url'].split('/')[-1] try: data = json.loads(row['data']) rdict[req] = data['result'][ 0] # we get back {'result': [workflow]} dict except Exception as exp: print("ERROR: fail to load data as json record, error=%s" % str(exp)) print(row) return rdict
def _getRequestWorkflows(self, requestNames): "Helper function to get all specs for given set of request names" urls = [str('%s/data/request/%s' % (self.msConfig['reqmgr2Url'], r)) for r in requestNames] self.logger.debug("getRequestWorkflows") for u in urls: self.logger.debug("url %s", u) data = multi_getdata(urls, ckey(), cert()) rdict = {} for row in data: req = row['url'].split('/')[-1] try: data = json.loads(row['data']) rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict except Exception as exp: self.logger.error("fail to process row %s", row) self.logger.exception("fail to load data as json record, error=%s", str(exp)) return rdict
def workqueueRequests(state=None): "Helper functions to get requests from WorkQueue" url = workqueueView('jobsByRequest') if state: pass # we may need to use state when we'll query WorkQueue params = {} headers = {'Accept': 'application/json'} mgr = RequestHandler() data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey()) data = json.loads(data) rdict = {} for row in data.get('rows', []): rdict[row['key']] = row['value'] return rdict
def agentsSites(url): "Return list of sites known in CMS WMAgents" sites_ready_in_agent = set() headers = {'Accept': 'application/json'} params = {} mgr = RequestHandler() res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert()) data = json.loads(res) agents = {} for r in [i['value'] for i in data['rows']]: team = r['agent_team'] if team != 'production': continue agents.setdefault(team, []).append(r) for team, agents in agents.items(): for agent in agents: if agent['status'] != 'ok': continue for site, sinfo in agent['WMBS_INFO']['thresholds'].iteritems(): if sinfo['state'] in ['Normal']: sites_ready_in_agent.add(site) return sites_ready_in_agent
def fetch(self): "Fetch information about sites from various CMS data-services" tfile = tempfile.NamedTemporaryFile() dashboardUrl = "http://dashb-ssb.cern.ch/dashboard/request.py" urls = [ '%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl, '%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl, ### FIXME: these calls to gwmsmon are failing pretty badly with ### "302 Found" and failing to decode, causing a huge error dump ### to the logs # 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary', # 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary', # 'https://cms-gwmsmon.cern.ch/poolview/json/totals', # 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus', 'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json', 'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt', 'http://cmsmonitoring.web.cern.ch/cmsmonitoring/storageoverview/latest/StorageOverview.json', ] cookie = {} ssbids = [ '106', '107', '108', '109', '136', '158', '159', '160', '237' ] sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6'] for url in urls: if 'gwmsmon' in url: cern_sso_cookie(url, tfile.name, cert(), ckey()) cookie.update({url: tfile.name}) gen = multi_getdata(urls, ckey(), cert(), cookie=cookie) siteInfo = {} for row in gen: if 'Detox' in row['url']: data = row['data'] else: try: data = json.loads(row['data']) except Exception as exc: self.logger.exception('error %s for row %s', str(exc), row) data = {} if 'ssb' in row['url']: for ssbid in ssbids: if ssbid in row['url']: siteInfo['ssb_%s' % ssbid] = data elif 'prodview/json/site_summary' in row['url']: siteInfo['gwmsmon_prod_site_summary'] = data elif 'totalview/json/site_summary' in row['url']: siteInfo['gwmsmon_site_summary'] = data elif 'totals' in row['url']: siteInfo['gwmsmon_totals'] = data elif 'maxusedcpus' in row['url']: siteInfo['gwmsmon_prod_maxused'] = data elif 'mcore' in row['url']: siteInfo['mcore'] = data elif 'Detox' in row['url']: siteInfo['detox_sites'] = data elif 'monitoring' in row['url']: siteInfo['mss_usage'] = data elif 'stuck' in row['url']: for sid in sids: if sid in row['url']: siteInfo['stuck_%s' % sid] = data siteInfo['site_queues'] = getNodeQueues() return siteInfo
def fetch(self): "Fetch information about sites from various CMS data-services" tfile = tempfile.NamedTemporaryFile() urls = [ '%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(), ### FIXME: these calls to gwmsmon are failing pretty badly with ### "302 Found" and failing to decode, causing a huge error dump ### to the logs # 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary', # 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary', # 'https://cms-gwmsmon.cern.ch/poolview/json/totals', # 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus', 'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json', 'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt', '%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(), ] cookie = {} ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237'] sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6'] for url in urls: if 'gwmsmon' in url: cern_sso_cookie(url, tfile.name, cert(), ckey()) cookie.update({url: tfile.name}) gen = multi_getdata(urls, ckey(), cert(), cookie=cookie) siteInfo = {} for row in gen: if 'Detox' in row['url']: data = row['data'] else: try: data = json.loads(row['data']) except Exception: traceback.print_exc() print(row) data = {} if 'ssb' in row['url']: for ssbid in ssbids: if ssbid in row['url']: siteInfo['ssb_%s' % ssbid] = data elif 'prodview/json/site_summary' in row['url']: siteInfo['gwmsmon_prod_site_summary'] = data elif 'totalview/json/site_summary' in row['url']: siteInfo['gwmsmon_site_summary'] = data elif 'totals' in row['url']: siteInfo['gwmsmon_totals'] = data elif 'maxusedcpus' in row['url']: siteInfo['gwmsmon_prod_maxused'] = data elif 'mcore' in row['url']: siteInfo['mcore'] = data elif 'Detox' in row['url']: siteInfo['detox_sites'] = data elif 'monitoring' in row['url']: siteInfo['mss_usage'] = data elif 'stuck' in row['url']: for sid in sids: if sid in row['url']: siteInfo['stuck_%s' % sid] = data siteInfo['site_queues'] = getNodeQueues() siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl()) return siteInfo
def fetch(self): "Fetch information about sites from various CMS data-services" tfile = tempfile.NamedTemporaryFile() urls = [ '%s/getplotdata?columnid=106&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=107&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=108&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=109&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=136&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=158&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=159&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=160&batch=1&lastdata=1' % dashboardUrl(), '%s/getplotdata?columnid=237&batch=1&lastdata=1' % dashboardUrl(), 'https://cms-gwmsmon.cern.ch/totalview/json/site_summary', 'https://cms-gwmsmon.cern.ch/prodview/json/site_summary', 'https://cms-gwmsmon.cern.ch/poolview/json/totals', 'https://cms-gwmsmon.cern.ch/prodview/json/maxusedcpus', 'http://cmsgwms-frontend-global.cern.ch/vofrontend/stage/mcore_siteinfo.json', 'http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/SitesInfo.txt', '%s/storageoverview/latest/StorageOverview.json' % monitoringUrl(), '%s/stuck_1.json' % stucktransferUrl(), '%s/stuck_2.json' % stucktransferUrl(), '%s/stuck_m1.json' % stucktransferUrl(), '%s/stuck_m3.json' % stucktransferUrl(), '%s/stuck_m4.json' % stucktransferUrl(), '%s/stuck_m5.json' % stucktransferUrl(), '%s/stuck_m6.json' % stucktransferUrl(), ] cookie = {} ssbids = ['106', '107', '108', '109', '136', '158', '159', '160', '237'] sids = ['1', '2', 'm1', 'm3', 'm4', 'm5', 'm6'] for url in urls: if 'gwmsmon' in url: cern_sso_cookie(url, tfile.name, cert(), ckey()) cookie.update({url: tfile.name}) gen = multi_getdata(urls, ckey(), cert(), cookie=cookie) siteInfo = {} for row in gen: if 'Detox' in row['url']: data = row['data'] else: try: data = json.loads(row['data']) except Exception: traceback.print_exc() print(row) data = {} if 'ssb' in row['url']: for ssbid in ssbids: if ssbid in row['url']: siteInfo['ssb_%s' % ssbid] = data elif 'prodview/json/site_summary' in row['url']: siteInfo['gwmsmon_prod_site_summary'] = data elif 'totalview/json/site_summary' in row['url']: siteInfo['gwmsmon_site_summary'] = data elif 'totals' in row['url']: siteInfo['gwmsmon_totals'] = data elif 'maxusedcpus' in row['url']: siteInfo['gwmsmon_prod_maxused'] = data elif 'mcore' in row['url']: siteInfo['mcore'] = data elif 'Detox' in row['url']: siteInfo['detox_sites'] = data elif 'monitoring' in row['url']: siteInfo['mss_usage'] = data elif 'stuck' in row['url']: for sid in sids: if sid in row['url']: siteInfo['stuck_%s' % sid] = data siteInfo['site_queues'] = getNodeQueues() siteInfo['ready_in_agent'] = agentsSites(agentInfoUrl()) return siteInfo