def get_data(url, kwargs=None, headers=None, verbose=None, decoder='json', post=False): "Retrive data" if not headers and url.find('DBSReader') != -1: headers = {'Accept': 'application/json' } # DBS3 always needs that ckey = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') try: # pycurl data look-up, primary way to get the data mgr = RequestHandler() with working_pem(PEMMGR.pem) as ckey: res = mgr.get_data(url, kwargs, headers, post, ckey, cert, verbose=verbose) if decoder == 'json': data = json.load(res) else: data = res.read() return data except Exception as exc: if verbose: print_error(exc) msg = 'Fall back to urllib' print_warning(msg) # urllib data look-up, fallback mechanism with working_pem(PEMMGR.pem) as ckey: return get_data_helper(url, kwargs, headers, verbose, decoder, post, ckey, cert)
def runsum(run, debug=0): "Get run information from RunSummary data-service" url = "https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary" args = {"DB": "cms_omds_lb", "FORMAT": "XML", "RUN": "%s" % run} url = run_summary_url(url, args) key = None cert = os.path.join(os.environ["HOME"], ".globus/usercert.pem") redirect = "https://cmswbm.web.cern.ch/Shibboleth.sso/ADFS" with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert, debug, redirect) for row in xml_parser(data, "runInfo"): yield row["runInfo"]
def runsum(run, debug=0): "Get run information from RunSummary data-service" url = "https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary" args = {"DB": "cms_omds_lb", "FORMAT": "XML", "RUN": "%s" % run} url = run_summary_url(url, args) key = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') redirect = 'https://cmswbm.web.cern.ch/Shibboleth.sso/ADFS' with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert, debug, redirect) for row in xml_parser(data, 'runInfo'): yield row['runInfo']
def get_data(url, kwargs=None, headers=None, verbose=None, decoder='json', post=False): "Retrive data" if not headers and url.find('DBSReader') != -1: headers = {'Accept': 'application/json'} # DBS3 always needs that ckey = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') try: # pycurl data look-up, primary way to get the data mgr = RequestHandler() with working_pem(PEMMGR.pem) as ckey: res = mgr.get_data(url, kwargs, headers, post, ckey, cert, verbose=verbose) if decoder == 'json': data = json.load(res) else: data = res.read() return data except Exception as exc: if verbose: print_error(exc) msg = 'Fall back to urllib' print_warning(msg) # urllib data look-up, fallback mechanism with working_pem(PEMMGR.pem) as ckey: return get_data_helper(url, kwargs, headers, verbose, decoder, post, ckey, cert)
def cms_vomsinit(_arg=None): """ cmssh command which executes voms-proxy-init on behalf of the user Examples: cmssh> vomsinit By default it applies the following options -rfc -voms cms:/cms -key <userkey.pem> -cert <usercert.pem> """ cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') with working_pem(PEMMGR.pem) as key: run("voms-proxy-destroy") cmd = "voms-proxy-init -rfc -voms cms:/cms -key %s -cert %s" % (key, cert) run(cmd) userdn = os.environ.get('USER_DN', '') if not userdn: cmd = "voms-proxy-info -identity" stdout, stderr = execmd(cmd) os.environ['USER_DN'] = stdout.replace('\n', '')
def prep(dataset): "Retrieve information from CMS ReqMgr data-service" dsn = dataset.split('/')[1] purl = 'http://cms.cern.ch/iCMS/jsp/mcprod/admin/requestmanagement.jsp' args = {'dsn': dsn, 'campid': 'any'} sso = 'https://cms.cern.ch/test/env.cgi?url=' url = sso + purl + '?' + urllib.urlencode(args) cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') data = '' # print "url", url with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert).read() params_dict, action = parse_sso_output(data) params = urllib.urlencode((params_dict)) # print "params", params # print "action", action if action: opener = create_https_opener(key, cert) fdesc = opener.open(action, params) data = fdesc.read() for row in data.split('\n'): if row.find('setCookie') != -1: ctup = row.split('(')[-1].replace('"', '').replace( "'", '').split(',')[:2] # print "key/val", ctup for hdl in opener.handlers: if repr(hdl).find('urllib2.HTTPCookieProcessor') != -1: for ccc in hdl.__dict__['cookiejar']: cookie = cookielib.Cookie(\ port=None, port_specified=False, domain=ccc.domain, domain_specified=False, domain_initial_dot=False, path=ccc.path, path_specified=False, secure=None, expires=None, discard=True, comment=None, comment_url=None, rest=None, version=0, name=ctup[0], value=ctup[1]) hdl.__dict__['cookiejar'].set_cookie(cookie) break print hdl.__dict__['cookiejar'] # print "\n### data", '\n'.join([r for r in data.split() if r]) fdesc = opener.open(purl + '?' + urllib.urlencode(args)) data = fdesc.read() # print "\n### data", data wrapwrite(html2text(data, ''))
def prep(dataset): "Retrieve information from CMS ReqMgr data-service" dsn = dataset.split('/')[1] purl= 'http://cms.cern.ch/iCMS/jsp/mcprod/admin/requestmanagement.jsp' args = {'dsn': dsn, 'campid':'any'} sso = 'https://cms.cern.ch/test/env.cgi?url=' url = sso + purl + '?' + urllib.urlencode(args) cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') data = '' # print "url", url with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert).read() params_dict, action = parse_sso_output(data) params = urllib.urlencode((params_dict)) # print "params", params # print "action", action if action: opener = create_https_opener(key, cert) fdesc = opener.open(action, params) data = fdesc.read() for row in data.split('\n'): if row.find('setCookie') != -1: ctup = row.split('(')[-1].replace('"', '').replace("'", '').split(',')[:2] # print "key/val", ctup for hdl in opener.handlers: if repr(hdl).find('urllib2.HTTPCookieProcessor') != -1: for ccc in hdl.__dict__['cookiejar']: cookie = cookielib.Cookie(\ port=None, port_specified=False, domain=ccc.domain, domain_specified=False, domain_initial_dot=False, path=ccc.path, path_specified=False, secure=None, expires=None, discard=True, comment=None, comment_url=None, rest=None, version=0, name=ctup[0], value=ctup[1]) hdl.__dict__['cookiejar'].set_cookie(cookie) break print hdl.__dict__['cookiejar'] # print "\n### data", '\n'.join([r for r in data.split() if r]) fdesc = opener.open(purl + '?' + urllib.urlencode(args)) data = fdesc.read() # print "\n### data", data wrapwrite(html2text(data, ''))
def reqmgr(dataset): "Retrieve information from CMS ReqMgr data-service" url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') configs = [] with working_pem(PEMMGR.pem) as key: with get_data_and_close(url, key, cert) as data: stream = data.read() try: jsondict = json.loads(stream) except Exception as _exc: jsondict = eval(stream, {"__builtins__": None}, {}) for key, val in jsondict.items(): for item in val: configs.append(item) if configs: print_info('Found %s configuration files:' % len(configs)) for rec in configs: print rec else: print_info('No configuration files is found')
def reqmgr(dataset): "Retrieve information from CMS ReqMgr data-service" url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') configs = [] with working_pem(PEMMGR.pem) as key: with get_data_and_close(url, key, cert) as data: stream = data.read() try: jsondict = json.loads(stream) except Exception as _exc: jsondict = eval(stream, { "__builtins__": None }, {}) for key, val in jsondict.items(): for item in val: configs.append(item) if configs: print_info('Found %s configuration files:' % len(configs)) for rec in configs: print rec else: print_info('No configuration files is found')
def init(self): "initialize SiteDB connection and retrieve all names" # get site names url = self.url + '/site-names' names = {} cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') with working_pem(PEMMGR.pem) as key: with get_data_and_close(url, key, cert) as data: for row in parser(data.read()): names[row['site_name']] = row['alias'] # get site resources url = self.url + '/site-resources' with get_data_and_close(url, key, cert) as data: for row in parser(data.read()): fqdn = row['fqdn'] for sename in row['fqdn'].split(','): self.mapping[sename.strip()] = names[row['site_name']] # get people info url = self.url + '/people' with get_data_and_close(url, key, cert) as data: for row in parser(data.read()): self.users[row['dn']] = row['username']
def read(url, output=None, debug=0): "Get run information from RunSummary data-service" encoding = 'utf-8' key = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') if os.path.isfile(url): with open(url, 'r') as stream: context = stream.read() try: pydoc.pager(context) except: print context return elif url.find('cmsweb.cern.ch') != -1: data = get_data(url, decoder=None) html = data encoding = None elif url.find('mcdb.cern.ch') != -1: data = urllib.urlopen(url) html = data.read().replace(' _place_holder;', '') encoding = enc(data.headers, html)[0] elif url.find('cern.ch') == -1: data = urllib.urlopen(url) html = data.read() encoding = enc(data.headers, html)[0] else: with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert, debug) html = data.read() encoding = enc(data.headers, html)[0] if encoding == 'us-ascii': encoding = 'utf-8' pager = os.environ.get('CMSSH_PAGER', None) if html: if int(os.environ.get('HTTPDEBUG', 0)): print_info('read data') print html if encoding: text = html.decode(encoding) res = html2text(text, '') if output: with open(output, 'w') as stream: stream.write(html) else: try: if pager: pydoc.pager(res.encode('utf-8')) else: wrapwrite(html2text(text, '')) except: wrapwrite(html2text(text, '')) else: if output: with open(output, 'w') as stream: stream.write(html) else: try: if pager: pydoc.pager(html) else: print html except: print html
def das_client(host, query, idx, limit, debug, dformat): "DAS client" if not query: raise Exception('You must provide input query') ckey = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') with working_pem(PEMMGR.pem) as ckey: data = get_data(host, query, idx, limit, debug, ckey, cert) if dformat == 'plain': jsondict = json.loads(data) if not jsondict.has_key('status'): print 'DAS record without status field:\n%s' % jsondict return if jsondict['status'] != 'ok': print "status: %s reason: %s" \ % (jsondict.get('status'), jsondict.get('reason', 'N/A')) return nres = jsondict['nresults'] if not limit: drange = '%s' % nres else: drange = '%s-%s out of %s' % (idx+1, idx+limit, nres) if limit: msg = "\nShowing %s results" % drange msg += ", for more results use --idx/--limit options\n" print msg mongo_query = jsondict['mongo_query'] unique = False fdict = mongo_query.get('filters', {}) filters = fdict.get('filters', []) aggregators = mongo_query.get('aggregators', []) if 'unique' in fdict.keys(): unique = True if filters and not aggregators: data = jsondict['data'] if isinstance(data, dict): rows = [r for r in get_value(data, filters)] print ' '.join(rows) elif isinstance(data, list): if unique: data = unique_filter(data) for row in data: rows = [r for r in get_value(row, filters)] print ' '.join(rows) else: print jsondict elif aggregators: data = jsondict['data'] if unique: data = unique_filter(data) for row in data: if row['key'].find('size') != -1 and \ row['function'] == 'sum': val = size_format(row['result']['value']) else: val = row['result']['value'] print '%s(%s)=%s' \ % (row['function'], row['key'], val) else: data = jsondict['data'] if isinstance(data, list): old = None val = None for row in data: val = prim_value(row) if not limit: if val != old: print val old = val else: print val if val != old and not limit: print val elif isinstance(data, dict): print prim_value(data) else: print data else: return json.loads(data)
def das_client(host, query, idx, limit, debug, dformat): "DAS client" if not query: raise Exception('You must provide input query') ckey = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') with working_pem(PEMMGR.pem) as ckey: data = get_data(host, query, idx, limit, debug, ckey, cert) if dformat == 'plain': jsondict = json.loads(data) if not jsondict.has_key('status'): print 'DAS record without status field:\n%s' % jsondict return if jsondict['status'] != 'ok': print "status: %s reason: %s" \ % (jsondict.get('status'), jsondict.get('reason', 'N/A')) return nres = jsondict['nresults'] if not limit: drange = '%s' % nres else: drange = '%s-%s out of %s' % (idx + 1, idx + limit, nres) if limit: msg = "\nShowing %s results" % drange msg += ", for more results use --idx/--limit options\n" print msg mongo_query = jsondict['mongo_query'] unique = False fdict = mongo_query.get('filters', {}) filters = fdict.get('filters', []) aggregators = mongo_query.get('aggregators', []) if 'unique' in fdict.keys(): unique = True if filters and not aggregators: data = jsondict['data'] if isinstance(data, dict): rows = [r for r in get_value(data, filters)] print ' '.join(rows) elif isinstance(data, list): if unique: data = unique_filter(data) for row in data: rows = [r for r in get_value(row, filters)] print ' '.join(rows) else: print jsondict elif aggregators: data = jsondict['data'] if unique: data = unique_filter(data) for row in data: if row['key'].find('size') != -1 and \ row['function'] == 'sum': val = size_format(row['result']['value']) else: val = row['result']['value'] print '%s(%s)=%s' \ % (row['function'], row['key'], val) else: data = jsondict['data'] if isinstance(data, list): old = None val = None for row in data: val = prim_value(row) if not limit: if val != old: print val old = val else: print val if val != old and not limit: print val elif isinstance(data, dict): print prim_value(data) else: print data else: return json.loads(data)