def get_data(url, kwargs=None, headers=None,
verbose=None, decoder='json', post=False):
"Retrive data"
if not headers and url.find('DBSReader') != -1:
headers = {'Accept': 'application/json' } # DBS3 always needs that
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
try:
# pycurl data look-up, primary way to get the data
mgr = RequestHandler()
with working_pem(PEMMGR.pem) as ckey:
res = mgr.get_data(url, kwargs, headers, post, ckey, cert, verbose=verbose)
if decoder == 'json':
data = json.load(res)
else:
data = res.read()
return data
except Exception as exc:
if verbose:
print_error(exc)
msg = 'Fall back to urllib'
print_warning(msg)
# urllib data look-up, fallback mechanism
with working_pem(PEMMGR.pem) as ckey:
return get_data_helper(url, kwargs, headers,
verbose, decoder, post, ckey, cert)
def runsum(run, debug=0):
"Get run information from RunSummary data-service"
url = "https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary"
args = {"DB": "cms_omds_lb", "FORMAT": "XML", "RUN": "%s" % run}
url = run_summary_url(url, args)
key = None
cert = os.path.join(os.environ["HOME"], ".globus/usercert.pem")
redirect = "https://cmswbm.web.cern.ch/Shibboleth.sso/ADFS"
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug, redirect)
for row in xml_parser(data, "runInfo"):
yield row["runInfo"]
def runsum(run, debug=0):
"Get run information from RunSummary data-service"
url = "https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary"
args = {"DB": "cms_omds_lb", "FORMAT": "XML", "RUN": "%s" % run}
url = run_summary_url(url, args)
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
redirect = 'https://cmswbm.web.cern.ch/Shibboleth.sso/ADFS'
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug, redirect)
for row in xml_parser(data, 'runInfo'):
yield row['runInfo']
def get_data(url,
kwargs=None,
headers=None,
verbose=None,
decoder='json',
post=False):
"Retrive data"
if not headers and url.find('DBSReader') != -1:
headers = {'Accept': 'application/json'} # DBS3 always needs that
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
try:
# pycurl data look-up, primary way to get the data
mgr = RequestHandler()
with working_pem(PEMMGR.pem) as ckey:
res = mgr.get_data(url,
kwargs,
headers,
post,
ckey,
cert,
verbose=verbose)
if decoder == 'json':
data = json.load(res)
else:
data = res.read()
return data
except Exception as exc:
if verbose:
print_error(exc)
msg = 'Fall back to urllib'
print_warning(msg)
# urllib data look-up, fallback mechanism
with working_pem(PEMMGR.pem) as ckey:
return get_data_helper(url, kwargs, headers, verbose, decoder,
post, ckey, cert)
def cms_vomsinit(_arg=None):
"""
cmssh command which executes voms-proxy-init on behalf of the user
Examples:
cmssh> vomsinit
By default it applies the following options
-rfc -voms cms:/cms -key <userkey.pem> -cert <usercert.pem>
"""
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as key:
run("voms-proxy-destroy")
cmd = "voms-proxy-init -rfc -voms cms:/cms -key %s -cert %s" % (key, cert)
run(cmd)
userdn = os.environ.get('USER_DN', '')
if not userdn:
cmd = "voms-proxy-info -identity"
stdout, stderr = execmd(cmd)
os.environ['USER_DN'] = stdout.replace('\n', '')
def cms_vomsinit(_arg=None):
"""
cmssh command which executes voms-proxy-init on behalf of the user
Examples:
cmssh> vomsinit
By default it applies the following options
-rfc -voms cms:/cms -key <userkey.pem> -cert <usercert.pem>
"""
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as key:
run("voms-proxy-destroy")
cmd = "voms-proxy-init -rfc -voms cms:/cms -key %s -cert %s" % (key,
cert)
run(cmd)
userdn = os.environ.get('USER_DN', '')
if not userdn:
cmd = "voms-proxy-info -identity"
stdout, stderr = execmd(cmd)
os.environ['USER_DN'] = stdout.replace('\n', '')
def prep(dataset):
"Retrieve information from CMS ReqMgr data-service"
dsn = dataset.split('/')[1]
purl = 'http://cms.cern.ch/iCMS/jsp/mcprod/admin/requestmanagement.jsp'
args = {'dsn': dsn, 'campid': 'any'}
sso = 'https://cms.cern.ch/test/env.cgi?url='
url = sso + purl + '?' + urllib.urlencode(args)
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
data = ''
# print "url", url
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert).read()
params_dict, action = parse_sso_output(data)
params = urllib.urlencode((params_dict))
# print "params", params
# print "action", action
if action:
opener = create_https_opener(key, cert)
fdesc = opener.open(action, params)
data = fdesc.read()
for row in data.split('\n'):
if row.find('setCookie') != -1:
ctup = row.split('(')[-1].replace('"', '').replace(
"'", '').split(',')[:2]
# print "key/val", ctup
for hdl in opener.handlers:
if repr(hdl).find('urllib2.HTTPCookieProcessor') != -1:
for ccc in hdl.__dict__['cookiejar']:
cookie = cookielib.Cookie(\
port=None, port_specified=False, domain=ccc.domain,
domain_specified=False, domain_initial_dot=False,
path=ccc.path, path_specified=False, secure=None, expires=None,
discard=True, comment=None, comment_url=None, rest=None,
version=0, name=ctup[0], value=ctup[1])
hdl.__dict__['cookiejar'].set_cookie(cookie)
break
print hdl.__dict__['cookiejar']
# print "\n### data", '\n'.join([r for r in data.split() if r])
fdesc = opener.open(purl + '?' + urllib.urlencode(args))
data = fdesc.read()
# print "\n### data", data
wrapwrite(html2text(data, ''))
def prep(dataset):
"Retrieve information from CMS ReqMgr data-service"
dsn = dataset.split('/')[1]
purl= 'http://cms.cern.ch/iCMS/jsp/mcprod/admin/requestmanagement.jsp'
args = {'dsn': dsn, 'campid':'any'}
sso = 'https://cms.cern.ch/test/env.cgi?url='
url = sso + purl + '?' + urllib.urlencode(args)
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
data = ''
# print "url", url
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert).read()
params_dict, action = parse_sso_output(data)
params = urllib.urlencode((params_dict))
# print "params", params
# print "action", action
if action:
opener = create_https_opener(key, cert)
fdesc = opener.open(action, params)
data = fdesc.read()
for row in data.split('\n'):
if row.find('setCookie') != -1:
ctup = row.split('(')[-1].replace('"', '').replace("'", '').split(',')[:2]
# print "key/val", ctup
for hdl in opener.handlers:
if repr(hdl).find('urllib2.HTTPCookieProcessor') != -1:
for ccc in hdl.__dict__['cookiejar']:
cookie = cookielib.Cookie(\
port=None, port_specified=False, domain=ccc.domain,
domain_specified=False, domain_initial_dot=False,
path=ccc.path, path_specified=False, secure=None, expires=None,
discard=True, comment=None, comment_url=None, rest=None,
version=0, name=ctup[0], value=ctup[1])
hdl.__dict__['cookiejar'].set_cookie(cookie)
break
print hdl.__dict__['cookiejar']
# print "\n### data", '\n'.join([r for r in data.split() if r])
fdesc = opener.open(purl + '?' + urllib.urlencode(args))
data = fdesc.read()
# print "\n### data", data
wrapwrite(html2text(data, ''))
def reqmgr(dataset):
"Retrieve information from CMS ReqMgr data-service"
url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
configs = []
with working_pem(PEMMGR.pem) as key:
with get_data_and_close(url, key, cert) as data:
stream = data.read()
try:
jsondict = json.loads(stream)
except Exception as _exc:
jsondict = eval(stream, {"__builtins__": None}, {})
for key, val in jsondict.items():
for item in val:
configs.append(item)
if configs:
print_info('Found %s configuration files:' % len(configs))
for rec in configs:
print rec
else:
print_info('No configuration files is found')
def reqmgr(dataset):
"Retrieve information from CMS ReqMgr data-service"
url = 'https://cmsweb.cern.ch/reqmgr/rest/configIDs' + dataset
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
configs = []
with working_pem(PEMMGR.pem) as key:
with get_data_and_close(url, key, cert) as data:
stream = data.read()
try:
jsondict = json.loads(stream)
except Exception as _exc:
jsondict = eval(stream, { "__builtins__": None }, {})
for key, val in jsondict.items():
for item in val:
configs.append(item)
if configs:
print_info('Found %s configuration files:' % len(configs))
for rec in configs:
print rec
else:
print_info('No configuration files is found')
def init(self):
"initialize SiteDB connection and retrieve all names"
# get site names
url = self.url + '/site-names'
names = {}
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as key:
with get_data_and_close(url, key, cert) as data:
for row in parser(data.read()):
names[row['site_name']] = row['alias']
# get site resources
url = self.url + '/site-resources'
with get_data_and_close(url, key, cert) as data:
for row in parser(data.read()):
fqdn = row['fqdn']
for sename in row['fqdn'].split(','):
self.mapping[sename.strip()] = names[row['site_name']]
# get people info
url = self.url + '/people'
with get_data_and_close(url, key, cert) as data:
for row in parser(data.read()):
self.users[row['dn']] = row['username']
def read(url, output=None, debug=0):
"Get run information from RunSummary data-service"
encoding = 'utf-8'
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(url):
with open(url, 'r') as stream:
context = stream.read()
try:
pydoc.pager(context)
except:
print context
return
elif url.find('cmsweb.cern.ch') != -1:
data = get_data(url, decoder=None)
html = data
encoding = None
elif url.find('mcdb.cern.ch') != -1:
data = urllib.urlopen(url)
html = data.read().replace(' _place_holder;', '')
encoding = enc(data.headers, html)[0]
elif url.find('cern.ch') == -1:
data = urllib.urlopen(url)
html = data.read()
encoding = enc(data.headers, html)[0]
else:
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug)
html = data.read()
encoding = enc(data.headers, html)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
pager = os.environ.get('CMSSH_PAGER', None)
if html:
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('read data')
print html
if encoding:
text = html.decode(encoding)
res = html2text(text, '')
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(res.encode('utf-8'))
else:
wrapwrite(html2text(text, ''))
except:
wrapwrite(html2text(text, ''))
else:
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(html)
else:
print html
except:
print html
def read(url, output=None, debug=0):
"Get run information from RunSummary data-service"
encoding = 'utf-8'
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(url):
with open(url, 'r') as stream:
context = stream.read()
try:
pydoc.pager(context)
except:
print context
return
elif url.find('cmsweb.cern.ch') != -1:
data = get_data(url, decoder=None)
html = data
encoding = None
elif url.find('mcdb.cern.ch') != -1:
data = urllib.urlopen(url)
html = data.read().replace(' _place_holder;', '')
encoding = enc(data.headers, html)[0]
elif url.find('cern.ch') == -1:
data = urllib.urlopen(url)
html = data.read()
encoding = enc(data.headers, html)[0]
else:
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug)
html = data.read()
encoding = enc(data.headers, html)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
pager = os.environ.get('CMSSH_PAGER', None)
if html:
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('read data')
print html
if encoding:
text = html.decode(encoding)
res = html2text(text, '')
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(res.encode('utf-8'))
else:
wrapwrite(html2text(text, ''))
except:
wrapwrite(html2text(text, ''))
else:
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(html)
else:
print html
except:
print html
def das_client(host, query, idx, limit, debug, dformat):
"DAS client"
if not query:
raise Exception('You must provide input query')
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as ckey:
data = get_data(host, query, idx, limit, debug, ckey, cert)
if dformat == 'plain':
jsondict = json.loads(data)
if not jsondict.has_key('status'):
print 'DAS record without status field:\n%s' % jsondict
return
if jsondict['status'] != 'ok':
print "status: %s reason: %s" \
% (jsondict.get('status'), jsondict.get('reason', 'N/A'))
return
nres = jsondict['nresults']
if not limit:
drange = '%s' % nres
else:
drange = '%s-%s out of %s' % (idx+1, idx+limit, nres)
if limit:
msg = "\nShowing %s results" % drange
msg += ", for more results use --idx/--limit options\n"
print msg
mongo_query = jsondict['mongo_query']
unique = False
fdict = mongo_query.get('filters', {})
filters = fdict.get('filters', [])
aggregators = mongo_query.get('aggregators', [])
if 'unique' in fdict.keys():
unique = True
if filters and not aggregators:
data = jsondict['data']
if isinstance(data, dict):
rows = [r for r in get_value(data, filters)]
print ' '.join(rows)
elif isinstance(data, list):
if unique:
data = unique_filter(data)
for row in data:
rows = [r for r in get_value(row, filters)]
print ' '.join(rows)
else:
print jsondict
elif aggregators:
data = jsondict['data']
if unique:
data = unique_filter(data)
for row in data:
if row['key'].find('size') != -1 and \
row['function'] == 'sum':
val = size_format(row['result']['value'])
else:
val = row['result']['value']
print '%s(%s)=%s' \
% (row['function'], row['key'], val)
else:
data = jsondict['data']
if isinstance(data, list):
old = None
val = None
for row in data:
val = prim_value(row)
if not limit:
if val != old:
print val
old = val
else:
print val
if val != old and not limit:
print val
elif isinstance(data, dict):
print prim_value(data)
else:
print data
else:
return json.loads(data)
def das_client(host, query, idx, limit, debug, dformat):
"DAS client"
if not query:
raise Exception('You must provide input query')
ckey = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
with working_pem(PEMMGR.pem) as ckey:
data = get_data(host, query, idx, limit, debug, ckey, cert)
if dformat == 'plain':
jsondict = json.loads(data)
if not jsondict.has_key('status'):
print 'DAS record without status field:\n%s' % jsondict
return
if jsondict['status'] != 'ok':
print "status: %s reason: %s" \
% (jsondict.get('status'), jsondict.get('reason', 'N/A'))
return
nres = jsondict['nresults']
if not limit:
drange = '%s' % nres
else:
drange = '%s-%s out of %s' % (idx + 1, idx + limit, nres)
if limit:
msg = "\nShowing %s results" % drange
msg += ", for more results use --idx/--limit options\n"
print msg
mongo_query = jsondict['mongo_query']
unique = False
fdict = mongo_query.get('filters', {})
filters = fdict.get('filters', [])
aggregators = mongo_query.get('aggregators', [])
if 'unique' in fdict.keys():
unique = True
if filters and not aggregators:
data = jsondict['data']
if isinstance(data, dict):
rows = [r for r in get_value(data, filters)]
print ' '.join(rows)
elif isinstance(data, list):
if unique:
data = unique_filter(data)
for row in data:
rows = [r for r in get_value(row, filters)]
print ' '.join(rows)
else:
print jsondict
elif aggregators:
data = jsondict['data']
if unique:
data = unique_filter(data)
for row in data:
if row['key'].find('size') != -1 and \
row['function'] == 'sum':
val = size_format(row['result']['value'])
else:
val = row['result']['value']
print '%s(%s)=%s' \
% (row['function'], row['key'], val)
else:
data = jsondict['data']
if isinstance(data, list):
old = None
val = None
for row in data:
val = prim_value(row)
if not limit:
if val != old:
print val
old = val
else:
print val
if val != old and not limit:
print val
elif isinstance(data, dict):
print prim_value(data)
else:
print data
else:
return json.loads(data)
