def test_yajl(self):
"""Test yajl wrapper"""
myjson.MODULE = 'yajl'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
data = {'a':1, 'b':2}
kwds = {'sort_keys':True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b':2, 'a':1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = 123
result1 = myjson.JSONEncoder(**kwds).encode(data)
kwds = {'sort_keys':True}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}}
result1 = myjson.JSONEncoder().encode(data)
result2 = myjson.JSONDecoder().decode(result1)
self.assertEqual(data, result2)
data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}}
result1 = myjson.dumps(data)
result2 = myjson.loads(result1)
self.assertEqual(data, result2)
myjson.MODULE = self.module
def send_request(self, method, kwargs):
"Send POST request to server with provided parameters"
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
format = getarg(kwargs, 'format', '')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
skey = getarg(kwargs, 'sort', '')
sdir = getarg(kwargs, 'dir', 'asc')
params = {'query':uinput, 'idx':idx, 'limit':limit,
'skey':skey, 'order':sdir}
if method == 'POST':
path = '/rest/create'
elif method == 'GET':
path = '/rest/request'
else:
raise Exception('Unsupported method %s' % method)
headers = {'Accept': 'application/json',
'Content-type': 'application/json'}
try:
data = urllib2_request(method, url+path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status':'fail', 'reason':traceback.format_exc()}
return result
def mongo_query(self):
"""
Read only mongo query, generated on demand.
"""
system = self._mongo_query.get('system', [])
filters = self._mongo_query.get('filters', {})
aggregators = self._mongo_query.get('aggregators', [])
if not self._mongo_query:
self._mongo_query = deepcopy(self.storage_query)
for key, val in self._mongo_query.items():
if key not in ['fields', 'spec']:
setattr(self, '_%s' % key, val)
spec = {}
for item in self._mongo_query.pop('spec'):
val = json.loads(item['value'])
if 'pattern' in item:
val = re.compile(val)
spec.update({item['key'] : val})
self._mongo_query['spec'] = spec
# special case when user asks for all records
fields = self._mongo_query.get('fields', None)
if fields and fields == ['records']:
self._mongo_query['fields'] = None
spec = {}
for key, val in self._mongo_query['spec'].items():
if key != 'records':
spec[key] = val
self._mongo_query = dict(fields=None, spec=spec)
if filters:
self._mongo_query.update({'filters':filters})
if aggregators:
self._mongo_query.update({'aggregators':aggregators})
if system:
self._mongo_query.update({'system':system})
return self._mongo_query
def send_request(self, method, kwargs):
"Send POST request to server with provided parameters"
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
format = getarg(kwargs, 'format', '')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
skey = getarg(kwargs, 'sort', '')
sdir = getarg(kwargs, 'dir', 'asc')
params = {
'query': uinput,
'idx': idx,
'limit': limit,
'skey': skey,
'order': sdir
}
if method == 'POST':
path = '/rest/create'
elif method == 'GET':
path = '/rest/request'
else:
raise Exception('Unsupported method %s' % method)
headers = {
'Accept': 'application/json',
'Content-type': 'application/json'
}
try:
data = urllib2_request(method, url + path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status': 'fail', 'reason': traceback.format_exc()}
return result
def get_file_run_lumis(url, api, args, verbose=0):
"Helper function to deal with file,run,lumi requests"
run_value = args.get("run_num", [])
if isinstance(run_value, dict) and "$in" in run_value:
runs = run_value["$in"]
elif isinstance(run_value, list):
runs = run_value
else:
if int_number_pattern.match(str(run_value)):
runs = [run_value]
elif run_value[0] == "[" and run_value[-1] == "]":
runs = json.loads(run_value)
else:
runs = []
args.update({"runs": runs})
blk = args.get("block_name", None)
if blk: # we don't need to look-up blocks
blocks = [blk]
else:
blocks = dbs_find("block", url, args, verbose)
if not blocks:
return
gen = file_run_lumis(url, blocks, runs, verbose)
key = "file_run"
if api.startswith("run_lumi"):
key = "run"
if api.startswith("file_lumi"):
key = "file"
if api.startswith("file_run_lumi"):
key = "file_run"
for row in process_lumis_with(key, gen):
yield row
def process(gen):
"Process generator from getdata"
for row in gen:
if "error" in row:
error = row.get("error")
reason = row.get("reason", "")
print dastimestamp("DAS ERROR"), error, reason
yield row
continue
if "data" in row:
yield json.loads(row["data"])
def process(gen):
"Process generator from getdata"
for row in gen:
if 'error' in row:
error = row.get('error')
reason = row.get('reason', '')
print(dastimestamp('DAS ERROR'), error, reason)
yield row
continue
if 'data' in row:
yield json.loads(row['data'])
def decode_mongo_query(query):
"""
Decode query from storage format into mongo format.
"""
spec = {}
for item in query.pop('spec'):
val = json.loads(item['value'])
if 'pattern' in item:
val = re.compile(val)
spec.update({item['key'] : val})
query['spec'] = spec
return query
def test_cjson(self):
"""Test cjson wrapper"""
myjson.MODULE = 'cjson'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
myjson.MODULE = self.module
def get_all_views(self, dbname=None):
"""
Method to get all degined views in couch db. The couch db doesn't have
a clear way to extract view documents. Instead we need to ask for
_all_docs and provide proper start/end-keys. Once we retrieve
_design docs, we loop over them and get the doc of particular view, e.g
http://localhost:5984/das/_design/dasviews
"""
if not dbname:
dbname = self.dbname
qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22'
host = 'http://' + self.uri
path = '/%s/_all_docs?%s' % (dbname, qqq)
kwds = {}
req = 'GET'
debug = 0
results = httplib_request(host, path, kwds, req, debug)
designdocs = json.loads(results)
results = {}
for item in designdocs['rows']:
doc = item['key']
# print "design:", doc
path = '/%s/%s' % (dbname, doc)
res = httplib_request(host, path, kwds, req, debug)
rdict = json.loads(res)
views = []
for view_name, view_dict in rdict['views'].items():
# print " view:", view_name
# print " map:", view_dict['map']
if 'reduce' in view_dict:
# print "reduce:", view_dict['reduce']
rdef = view_dict['reduce']
defrow = dict(map=view_dict['map'],
reduce=view_dict['reduce'])
else:
defrow = dict(map=view_dict['map'])
row = {'%s' % view_name: defrow}
views.append(row)
results[doc] = views
return results
def get_all_views(self, dbname=None):
"""
Method to get all degined views in couch db. The couch db doesn't have
a clear way to extract view documents. Instead we need to ask for
_all_docs and provide proper start/end-keys. Once we retrieve
_design docs, we loop over them and get the doc of particular view, e.g
http://localhost:5984/das/_design/dasviews
"""
if not dbname:
dbname = self.dbname
qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22'
host = 'http://' + self.uri
path = '/%s/_all_docs?%s' % (dbname, qqq)
kwds = {}
req = 'GET'
debug = 0
results = httplib_request(host, path, kwds, req, debug)
designdocs = json.loads(results)
results = {}
for item in designdocs['rows']:
doc = item['key']
# print "design:", doc
path = '/%s/%s' % (dbname, doc)
res = httplib_request(host, path, kwds, req, debug)
rdict = json.loads(res)
views = []
for view_name, view_dict in rdict['views'].items():
# print " view:", view_name
# print " map:", view_dict['map']
if 'reduce' in view_dict:
# print "reduce:", view_dict['reduce']
rdef = view_dict['reduce']
defrow = dict(map=view_dict['map'],
reduce=view_dict['reduce'])
else:
defrow = dict(map=view_dict['map'])
row = {'%s' % view_name : defrow}
views.append(row)
results[doc] = views
return results
def configs(url, args, verbose=False):
"""Find config info in ReqMgr"""
headers = {'Accept': 'application/json;text/json'}
dataset = args.get('dataset', None)
if not dataset:
return
base = 'https://%s' % url.split('/')[2]
idict, source = findReqMgrIds(dataset, base, verbose)
ids = []
ids_types = {} # keep track of ids/types relationship
for key, ilist in idict.items():
rtype = 'output' if key.lower().find('output') != -1 else 'input'
for item in ilist:
ids.append(item)
ids_types[item] = rtype
# for hash ids find configs via ReqMgr REST API
urls = [rurl(base, i) for i in ids if len(i) == 32]
# for non-hash ids probe to find configs in showWorkload
req_urls = ['%s/couchdb/reqmgr_workload_cache/%s' \
% (base, i) for i in ids if len(i) != 32]
if req_urls:
gen = urlfetch_getdata(req_urls, CKEY, CERT, headers)
config_urls = []
for row in gen:
if 'error' not in row:
url = row['url']
for key, rtype in ids_types.items():
if key in url:
break
rdict = json.loads(row['data'])
for key in rdict.keys():
val = rdict[key]
if key.endswith('ConfigCacheID'):
if isinstance(val, basestring):
config_urls.append(rurl(base, val))
ids_types[val] = rtype
elif isinstance(val, dict):
for kkk in val.keys():
if kkk.endswith('ConfigCacheID'):
vvv = val[kkk]
if isinstance(vvv, basestring):
config_urls.append(rurl(base, vvv))
ids_types[vvv] = rtype
if config_urls:
urls += config_urls
udict = {}
for rid, rtype in ids_types.items():
for url in set(urls):
if rid in url:
udict.setdefault(rtype, []).append(url)
config = {'dataset':dataset, 'name': source, 'urls': udict, 'ids': ids, 'idict': idict}
yield {'config': config}
def _test_yajl(self):
"""Test yajl wrapper"""
import yajl
myjson.MODULE = 'yajl'
myjson.yajl = yajl # so not to depend on import sequence
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
data = {'a': 1, 'b': 2}
kwds = {'sort_keys': True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b': 2, 'a': 1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = 123
result1 = myjson.JSONEncoder(**kwds).encode(data)
kwds = {'sort_keys': True}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}}
result1 = myjson.JSONEncoder().encode(data)
result2 = myjson.JSONDecoder().decode(result1)
self.assertEqual(data, result2)
data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}}
result1 = myjson.dumps(data)
result2 = myjson.loads(result1)
self.assertEqual(data, result2)
myjson.MODULE = self.module
def status(self, **kwargs):
"""
Place request to obtain status about given query
"""
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
req = """
<script type="application/javascript">
setTimeout('ajaxStatus()',3000)
</script>"""
def set_header():
"Set HTTP header parameters"
tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
cherrypy.response.headers['Expire'] = tstamp
cherrypy.response.headers['Cache-control'] = 'no-cache'
uinput = kwargs.get('input', '')
uinput = urllib.unquote_plus(uinput)
ajax = kwargs.get('ajax', 1)
view = kwargs.get('view', 'list')
params = {'query':uinput}
path = '/rest/status'
url = self.cachesrv
headers = {'Accept': 'application/json'}
try:
res = urllib2_request('GET', url+path, params, headers=headers)
data = json.loads(res)
except:
self.daslogger.error(traceback.format_exc())
data = {'status':'fail'}
if ajax:
cherrypy.response.headers['Content-Type'] = 'text/xml'
if data['status'] == 'ok':
page = '<script type="application/javascript">reload()</script>'
elif data['status'] == 'fail':
page = '<script type="application/javascript">reload()</script>'
page += self.error(self.gen_error_msg(kwargs))
else:
page = img + ' ' + str(data['status']) + ', please wait...'
img_stop = ''
page += ', <a href="/das/">stop</a> request'
page += req
set_header()
page = ajax_response(page)
else:
try:
page = data['status']
except:
page = traceback.format_exc()
return page
def result(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
result = self.send_request("GET", kwargs)
res = []
if type(result) is types.StringType:
data = json.loads(result)
else:
data = result
if data["status"] == "success":
res = data["data"]
return res
def status(self, **kwargs):
"""
Place request to obtain status about given query
"""
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
req = """
<script type="application/javascript">
setTimeout('ajaxStatus()',3000)
</script>"""
def set_header():
"Set HTTP header parameters"
tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
cherrypy.response.headers["Expire"] = tstamp
cherrypy.response.headers["Cache-control"] = "no-cache"
uinput = kwargs.get("input", "")
uinput = urllib.unquote_plus(uinput)
ajax = kwargs.get("ajax", 1)
view = kwargs.get("view", "list")
params = {"query": uinput}
path = "/rest/status"
url = self.cachesrv
headers = {"Accept": "application/json"}
try:
res = urllib2_request("GET", url + path, params, headers=headers)
data = json.loads(res)
except:
self.daslogger.error(traceback.format_exc())
data = {"status": "fail"}
if ajax:
cherrypy.response.headers["Content-Type"] = "text/xml"
if data["status"] == "ok":
page = '<script type="application/javascript">reload()</script>'
elif data["status"] == "fail":
page = '<script type="application/javascript">reload()</script>'
page += self.error(self.gen_error_msg(kwargs))
else:
page = img + " " + str(data["status"]) + ", please wait..."
img_stop = ""
page += ', <a href="/das/">stop</a> request'
page += req
set_header()
page = ajax_response(page)
else:
try:
page = data["status"]
except:
page = traceback.format_exc()
return page
def status(self, **kwargs):
"""
Place request to obtain status about given query
"""
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
req = """
<script type="application/javascript">
setTimeout('ajaxStatus()',3000)
</script>"""
def set_header():
"Set HTTP header parameters"
tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
cherrypy.response.headers['Expire'] = tstamp
cherrypy.response.headers['Cache-control'] = 'no-cache'
uinput = kwargs.get('input', '')
uinput = urllib.unquote_plus(uinput)
ajax = kwargs.get('ajax', 1)
view = kwargs.get('view', 'list')
params = {'query': uinput}
path = '/rest/status'
url = self.cachesrv
headers = {'Accept': 'application/json'}
try:
res = urllib2_request('GET', url + path, params, headers=headers)
data = json.loads(res)
except:
self.daslogger.error(traceback.format_exc())
data = {'status': 'fail'}
if ajax:
cherrypy.response.headers['Content-Type'] = 'text/xml'
if data['status'] == 'ok':
page = '<script type="application/javascript">reload()</script>'
elif data['status'] == 'fail':
page = '<script type="application/javascript">reload()</script>'
page += self.error(self.gen_error_msg(kwargs))
else:
page = img + ' ' + str(data['status']) + ', please wait...'
img_stop = ''
page += ', <a href="/das/">stop</a> request'
page += req
set_header()
page = ajax_response(page)
else:
try:
page = data['status']
except:
page = traceback.format_exc()
return page
def result(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
result = self.send_request('GET', kwargs)
res = []
if type(result) is bytes:
data = json.loads(result)
else:
data = result
if data['status'] == 'success':
res = data['data']
return res
def result(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
result = self.send_request('GET', kwargs)
res = []
if type(result) is bytes:
data = json.loads(result)
else:
data = result
if data['status'] == 'success':
res = data['data']
return res
def phedex_info(urls, dbsdata):
"Get phedex info for given set of dbs data"
# create list of URLs for urlfetch
url = urls.get('phedex') + '/blockReplicas'
urls = ('%s?dataset=%s' % (url, d) for d in dbsdata.keys())
headers = {'Accept':'application/json;text/json'}
gen = urlfetch_getdata(urls, CKEY, CERT, headers)
for ddict in gen:
try:
jsondict = json.loads(ddict['data'])
except Exception as _exc:
continue
rec = {}
for blk in jsondict['phedex']['block']:
dataset = blk['name'].split('#')[0]
if 'nfiles' not in rec:
nfiles = blk['files']
size = blk['bytes']
else:
nfiles = rec['nfiles'] + blk['files']
size = rec['size'] + blk['bytes']
rec.update({'nfiles':nfiles, 'size':size})
for rep in blk['replica']:
if 'site' not in rec:
rec = dict(dataset=dataset, nfiles=nfiles, size=size,
site=[rep['node']], se=[rep['se']],
custodial=[rep['custodial']])
rec.update(dbsdata[dataset])
else:
sites = rec['site']
ses = rec['se']
custodial = rec['custodial']
if rep['node'] not in sites:
sites.append(rep['node'])
ses.append(rep['se'])
custodial.append(rep['custodial'])
rec.update({'site':sites, 'se':ses, 'custodial':custodial})
if rec:
# unwrap the site/se/custodial lists and yield records w/ their
# individual values
for idx in range(0, len(rec['site'])):
sename = rec['se'][idx]
site = rec['site'][idx]
custodial = rec['custodial'][idx]
newrec = dict(rec)
newrec['se'] = sename
newrec['site'] = site
newrec['custodial'] = custodial
yield newrec
def _test_cjson(self):
""" Test cjson wrapper """
import cjson
myjson.MODULE = 'cjson'
myjson.cjson = cjson
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
myjson.MODULE = self.module
def block_summary(dbs, blocks):
"Get block summary information for given set of blocks"
headers = {"Accept": "text/json;application/json"}
url = dbs + "/blocksummaries"
urls = ["%s/?block_name=%s" % (url, urllib.quote(b)) for b in blocks]
res = urlfetch_getdata(urls, CKEY, CERT, headers)
for row in res:
if "error" in row:
error = row.get("error")
reason = row.get("reason", "")
yield {"error": error, "reason": reason}
continue
url = row["url"]
blk = urllib.unquote(url.split("=")[-1])
for rec in json.loads(row["data"]):
data = {"name": blk, "size": rec["file_size"], "nfiles": rec["num_file"], "nevents": rec["num_event"]}
yield dict(block=data)
def test_json(self):
"""Test json wrapper"""
myjson.MODULE = 'json'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
data = {'a':1, 'b':2}
kwds = {'sort_keys':True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b':2, 'a':1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
myjson.MODULE = self.module
def records(self, *args, **kwargs):
"""
HTTP GET request.
Retrieve records from provided collection.
"""
data = {'server_method': 'request'}
if 'query' not in kwargs:
data['status'] = 'fail'
data['reason'] = 'no query is provided'
return data
# input query in JSON format, we should decode it using json.
query = json.loads(kwargs.get('query'))
coll = kwargs.get('collection', 'merge')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion
count = kwargs.get('count', 0)
data.update({
'status': 'requested',
'query': kwargs['query'],
'collection': coll,
'count': count
})
if '_id' in query['spec']:
recid = query['spec']['_id']
ids = []
if type(recid) is bytes:
ids = [ObjectId(recid)]
elif type(recid) is list:
ids = [ObjectId(r) for r in recid]
spec = {'spec': {'_id': {'$in': ids}}}
else: # look-up all records
spec = {}
self.logdb(query)
try:
gen = self.dascore.rawcache.get_from_cache\
(spec, idx=idx, limit=limit, collection=coll, adjust=False)
data['status'] = 'success'
data['data'] = [r for r in gen]
except:
self.debug(traceback.format_exc())
data['status'] = 'fail'
data['reason'] = sys.exc_info()[0]
return data
def block_summary(dbs, blocks):
"Get block summary information for given set of blocks"
headers = {'Accept':'text/json;application/json'}
url = dbs + "/blocksummaries"
urls = ['%s/?block_name=%s' % (url, urllib.quote(b)) for b in blocks]
res = urlfetch_getdata(urls, CKEY, CERT, headers)
for row in res:
if 'error' in row:
error = row.get('error')
reason = row.get('reason', '')
yield {'error':error, 'reason':reason}
continue
url = row['url']
blk = urllib.unquote(url.split('=')[-1])
for rec in json.loads(row['data']):
data = {'name': blk, 'size': rec['file_size'],
'nfiles': rec['num_file'],
'nevents': rec['num_event']}
yield dict(block=data)
def _test_json(self):
""" Test json wrapper """
myjson.MODULE = 'json'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
data = {'a': 1, 'b': 2}
kwds = {'sort_keys': True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b': 2, 'a': 1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
myjson.MODULE = self.module
def extract_http_error(err):
"""
Upon urllib failure the data-service can send HTTPError message.
It can be in a form of JSON, etc. This function attempts to extract
such message. If it fails it just str(err) and return.
"""
msg = str(err)
try:
err = json.loads(err)
if err.has_key('message'):
value = err['message']
if isinstance(value, dict):
msg = ''
for key, val in value.iteritems():
msg += '%s: %s. ' % (key, val)
else:
msg = str(value)
except:
pass
return msg
def worker_v2(url, query):
"""
Query RunRegistry service, see documentation at
https://twiki.cern.ch/twiki/bin/viewauth/CMS/DqmRrApi
url=http://pccmsdqm04.cern.ch/runregistry/xmlrpc
"""
server = xmlrpclib.ServerProxy(url)
namespace = 'GLOBAL'
if isinstance(query, str) or isinstance(query, unicode):
try:
data = server.RunLumiSectionRangeTable.exportJson(namespace, query)
except Exception as _err:
data = "{}" # empty response
for row in json.loads(data):
yield row
elif isinstance(query, dict):
iformat = 'tsv_runs' # other formats are xml_all, csv_runs
try:
data = server.RunDatasetTable.export(namespace, iformat, query)
except Exception as _err:
data = "" # empty response
titles = []
for line in data.split('\n'):
if not line:
continue
if not titles:
for title in line.split('\t')[:-1]:
title = title.lower()
if title != 'run_number':
title = title.replace('run_', '')
titles.append(title)
continue
val = line.split('\t')[:-1]
if len(val) != len(titles):
continue
record = {}
for idx in range(0, len(titles)):
key = titles[idx]
record[key] = adjust_value(val[idx])
yield dict(run=record)
def rest(self, *args, **kwargs):
"""
RESTful interface. We use args tuple as access method(s), e.g.
args = ('method',) and kwargs to represent input parameters.
"""
request = cherrypy.request.method
if request not in self.methods.keys():
msg = "Usupported request '%s'" % requset
return {'error': msg}
method = args[0]
if method not in self.methods[request].keys():
msg = "Unsupported method '%s'" % method
return {'error': msg}
if request == 'POST':
if cherrypy.request.body:
body = cherrypy.request.body.read()
try:
kwargs = json.loads(body)
except:
msg = "Unable to load body request"
return {'error': msg}
return getattr(self, method)(kwargs)
def nresults(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
params = {'query': uinput}
path = '/rest/nresults'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url + path, params, headers=headers)
record = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
record = {'status': 'fail', 'reason': traceback.format_exc()}
if record['status'] == 'success':
return record['nresults']
else:
msg = "nresults returns status: %s" % str(record)
self.daslogger.info(msg)
return -1
def nresults(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
url = self.cachesrv
uinput = getarg(kwargs, "input", "")
params = {"query": uinput}
path = "/rest/nresults"
headers = {"Accept": "application/json"}
try:
data = urllib2_request("GET", url + path, params, headers=headers)
record = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
record = {"status": "fail", "reason": traceback.format_exc()}
if record["status"] == "success":
return record["nresults"]
else:
msg = "nresults returns status: %s" % str(record)
self.daslogger.info(msg)
return -1
def nresults(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
params = {'query':uinput}
path = '/rest/nresults'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url+path, params, headers=headers)
record = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
record = {'status':'fail', 'reason':traceback.format_exc()}
if record['status'] == 'success':
return record['nresults']
else:
msg = "nresults returns status: %s" % str(record)
self.daslogger.info(msg)
return -1
def rest(self, *args, **kwargs):
"""
RESTful interface. We use args tuple as access method(s), e.g.
args = ('method',) and kwargs to represent input parameters.
"""
request = cherrypy.request.method
if request not in self.methods.keys():
msg = "Usupported request '%s'" % requset
return {'error': msg}
method = args[0]
if method not in self.methods[request].keys():
msg = "Unsupported method '%s'" % method
return {'error': msg}
if request == 'POST':
if cherrypy.request.body:
body = cherrypy.request.body.read()
try:
kwargs = json.loads(body)
except:
msg = "Unable to load body request"
return {'error': msg}
return getattr(self, method)(kwargs)
def records(self, *args, **kwargs):
"""
HTTP GET request.
Retrieve records from provided collection.
"""
data = {'server_method':'request'}
if not kwargs.has_key('query'):
data['status'] = 'fail'
data['reason'] = 'no query is provided'
return data
# input query in JSON format, we should decode it using json.
query = json.loads(kwargs.get('query'))
coll = kwargs.get('collection', 'merge')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion
count = kwargs.get('count', 0)
data.update({'status':'requested', 'query':kwargs['query'],
'collection':coll, 'count': count})
if query['spec'].has_key('_id'):
recid = query['spec']['_id']
ids = []
if type(recid) is types.StringType:
ids = [ObjectId(recid)]
elif type(recid) is types.ListType:
ids = [ObjectId(r) for r in recid]
spec = {'spec':{'_id':{'$in':ids}}}
else: # look-up all records
spec = {}
self.logdb(query)
try:
gen = self.dascore.rawcache.get_from_cache\
(spec, idx=idx, limit=limit, collection=coll, adjust=False)
data['status'] = 'success'
data['data'] = [r for r in gen]
except:
self.debug(traceback.format_exc())
data['status'] = 'fail'
data['reason'] = sys.exc_type
return data
def get_file_run_lumis(url, api, args, verbose=0):
"Helper function to deal with file,run,lumi requests"
run_value = args.get('run_num', [])
if isinstance(run_value, dict) and '$in' in run_value:
runs = run_value['$in']
elif isinstance(run_value, list):
runs = run_value
else:
if int_number_pattern.match(str(run_value)):
runs = [run_value]
elif run_value[0]=='[' and run_value[-1]==']':
if '-' in run_value: # continuous range
runs = run_value.replace("'", '').replace('[', '').replace(']', '')
else:
runs = json.loads(run_value)
else:
runs = run_value
args.update({'runs': runs})
blk = args.get('block_name', None)
if blk: # we don't need to look-up blocks
blocks = [blk]
else:
blocks = dbs_find('block', url, args, verbose)
if not blocks:
return
valid = 1 if args.get('validFileOnly', '') else 0
gen = file_run_lumis(url, blocks, runs, valid, verbose)
key = 'file_run'
if api.startswith('run_lumi'):
key = 'run'
if api.startswith('file_lumi'):
key = 'file'
if api.startswith('file_run'):
key = 'file_run'
if api.startswith('file_run_lumi'):
key = 'file_run'
for row in process_lumis_with(key, gen):
yield row
def parser(self, dasquery, dformat, source, args):
"""
Data parser for Monitor service.
"""
if isinstance(source, InstanceType) or isinstance(source, file):
try: # we got data descriptor
data = source.read()
except:
source.close()
raise
source.close()
elif isinstance(source, object) and hasattr(source, 'read'): # StringIO
data = source.read()
source.close()
else:
data = source
try:
row = json.loads(data)
except:
msg = "MonitorService::parser,"
msg += " WARNING, fail to JSON'ify data:\n%s" % data
self.logger.warning(msg)
row = eval(data, { "__builtins__": None }, {})
try:
monitor_time = row['series']
monitor_data = row['data']
items = ({'time':list(t), 'data':d} for t, d in \
zip(monitor_time, monitor_data))
for row in items:
interval = row['time']
dataval = row['data']
for key, val in dataval.iteritems():
newrow = {'time': interval}
newrow[args['grouping']] = key
newrow['rate'] = val
yield dict(monitor=newrow)
except:
yield dict(monitor=row)
def file_run_lumis(url, blocks, runs=None, valid=None, verbose=0):
"""
Find file, run, lumi tuple for given set of files and (optional) runs.
"""
headers = {'Accept': 'application/json;text/json'}
urls = []
for blk in blocks:
if not blk:
continue
dbs_url = '%s/filelumis/?block_name=%s' % (url, urllib.quote(blk))
if valid:
dbs_url += '&validFileOnly=1'
if runs:
dbs_url += "&run_num=%s" % urllib.quote(str(runs))
urls.append(dbs_url)
if not urls:
return
if verbose > 1:
print("\nDEBUG: file_run_lumis")
print(urls)
gen = urlfetch_getdata(urls, CKEY, CERT, headers)
odict = {} # output dict
for rec in gen:
if 'error' in rec:
error = rec.get('error')
reason = rec.get('reason', '')
yield {'error':error, 'reason':reason}
else:
for row in json.loads(rec['data']):
run = row['run_num']
lfn = row['logical_file_name']
lumilist = row['lumi_section_num']
key = (lfn, run)
for lumi in lumilist:
odict.setdefault(key, []).append(lumi)
for key, lumis in odict.items():
lfn, run = key
yield lfn, run, lumis
def urlfetch_proxy(urls):
"Proxy client for Go proxy server"
params = {'urls': '\n'.join(urls)}
encoded_data = urllib.urlencode(params)
server = "http://localhost:8215/fetch"
req = urllib2.Request(server)
data = urllib2.urlopen(req, encoded_data)
code = data.getcode()
if code == 200:
if not urls: # ping request
yield {'ping':'pong'}
else:
for row in data.readlines():
if row and row[0] == '{' and row.find('data'):
# Erlang response
rec = json.loads(row)
for line in rec['data'].split('\n'):
yield line
else:
# Go response
yield row
else:
yield {'error':'Fail to contact UrlFetch proxy server', 'code':code}
def urlfetch_proxy(urls):
"Proxy client for Go proxy server"
params = {"urls": "\n".join(urls)}
encoded_data = urllib.urlencode(params)
server = "http://localhost:8215/fetch"
req = urllib2.Request(server)
data = urllib2.urlopen(req, encoded_data)
code = data.getcode()
if code == 200:
if not urls: # ping request
yield {"ping": "pong"}
else:
for row in data.readlines():
if row and row[0] == "{" and row.find("data"):
# Erlang response
rec = json.loads(row)
for line in rec["data"].split("\n"):
yield line
else:
# Go response
yield row
else:
yield {"error": "Fail to contact UrlFetch proxy server", "code": code}
def block_run_lumis(url, blocks, runs=None, verbose=0):
"""
Find block, run, lumi tuple for given set of files and (optional) runs.
"""
headers = {"Accept": "application/json;text/json"}
urls = []
params = {}
for blk in blocks:
if not blk:
continue
dbs_url = "%s/filelumis/?block_name=%s" % (url, urllib.quote(blk))
if runs and isinstance(runs, list):
params.update({"run_num": runrange(runs[0], runs[-1], False)})
urls.append(dbs_url)
if not urls:
return
if verbose > 1:
print "\nDEBUG: block_run_lumis"
print urls
gen = urlfetch_getdata(urls, CKEY, CERT, headers)
odict = {} # output dict
for rec in gen:
blk = urllib.unquote(url_args(rec["url"])["block_name"])
if "error" in rec:
error = rec.get("error")
reason = rec.get("reason", "")
yield {"error": error, "reason": reason}
else:
for row in json.loads(rec["data"]):
run = row["run_num"]
lumilist = row["lumi_section_num"]
key = (blk, run)
for lumi in lumilist:
odict.setdefault(key, []).append(lumi)
for key, lumis in odict.iteritems():
blk, run = key
yield blk, run, lumis
def wrapper(self, *args, **kwds):
"""Wrapper for decorator"""
# check request headers. For methods POST/PUT
# we need to read request body to get parameters
headers = cherrypy.request.headers
if cherrypy.request.method == 'POST' or\
cherrypy.request.method == 'PUT':
body = cherrypy.request.body.read()
if args and kwds:
msg = 'Misleading request.\n'
msg += 'Request: %s\n' % cherrypy.request.method
msg += 'Headers: %s\n' % headers
msg += 'Parameters: args=%s, kwds=%s\n' % (args, kwds)
return {'status': 'fail', 'reason': msg}
if body:
jsondict = json.loads(body, encoding='latin-1')
else:
jsondict = kwds
for key, val in jsondict.items():
kwds[str(key)] = str(val)
# headers = cherrypy.request.headers
# if headers.has_key('Content-type'):
# content = headers['Content-type']
# cherrypy.response.headers['Content-type'] = content
# if content in ['application/json', 'text/json', 'text/x-json']:
# body = cherrypy.request.body.read()
# if args and kwds:
# msg = 'Misleading request.'
# msg += 'Headers: %s ' % headers
# msg += 'Parameters: %s, %s' % (args, kwds)
# return {'status':'fail', 'reason': msg}
# jsondict = json.loads(body, encoding='latin-1')
# for key, val in jsondict.items():
# kwds[str(key)] = str(val)
pat = re.compile('^[+]?\d*$')
supported = [
'query', 'idx', 'limit', 'expire', 'method', 'skey', 'order',
'collection'
]
if not kwds:
if args:
kwds = args[-1]
keys = []
if kwds:
keys = [i for i in kwds.keys() if i not in supported]
if keys:
msg = 'Unsupported keys: %s' % keys
return {'status': 'fail', 'reason': msg}
if 'idx' in kwds and not pat.match(str(kwds['idx'])):
msg = 'Unsupported value idx=%s' % (kwds['idx'])
return {'status': 'fail', 'reason': msg}
if 'limit' in kwds and not pat.match(str(kwds['limit'])):
msg = 'Unsupported value limit=%s' % (kwds['limit'])
return {'status': 'fail', 'reason': msg}
if 'expire' in kwds and not pat.match(str(kwds['expire'])):
msg = 'Unsupported value expire=%s' % (kwds['expire'])
return {'status': 'fail', 'reason': msg}
if 'order' in kwds:
if kwds['order'] not in ['asc', 'desc']:
msg = 'Unsupported value order=%s' % (kwds['order'])
return {'status': 'fail', 'reason': msg}
data = func(self, *args, **kwds)
return data
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ''
if args:
recordid = args[0]
spec = {'_id': recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and '_id' in kwargs:
spec = {'_id': kwargs['_id']}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
coll = getarg(kwargs, 'collection', 'merge')
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {
'query': json.dumps(query),
'idx': idx,
'limit': limit,
'collection': coll
}
path = '/rest/records'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET',
url + path,
params,
headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status': 'fail', 'reason': traceback.format_exc()}
res = ""
if result['status'] == 'success':
if recordid: # we got id
for row in result['data']:
if show == 'json':
jsoncode = {'jsoncode': json2html(row, "")}
res += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(row, indent=1, width=100)
res += self.templatepage('das_code', code=code)
else:
code = yaml.dump(row,
width=100,
indent=4,
default_flow_style=False)
res += self.templatepage('das_code', code=code)
else:
for row in result['data']:
rid = row['_id']
del row['_id']
record = dict(id=rid, daskeys=', '.join(row))
res += self.templatepage('das_record', **record)
else:
res = result['status']
if 'reason' in res:
return self.error(res['reason'])
else:
msg = 'Uknown error, kwargs=' % kwargs
return self.error(msg)
if recordid:
if format:
if format == 'xml':
return self.wrap2dasxml(result['data'])
elif format == 'json':
return self.wrap2dasjson(result['data'])
else:
return self.error('Unsupported data format %s' %
format)
page = res
else:
url = '/das/records?'
idict = dict(nrows=nresults,
idx=idx,
limit=limit,
results=res,
url=url)
page = self.templatepage('das_pagination', **idict)
form = self.form(uinput="")
ctime = (time.time() - time0)
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def requestquery(self, query, add_to_analytics=True):
"""
Query analyzer which form request query to DAS from a free text-based form.
Return MongoDB request query.
"""
# strip operators while we will match words against them
operators = [o.strip() for o in self.operators]
# find out if input query contains filters/mapreduce functions
mapreduce = []
filters = []
aggregators = []
pat = re.compile(r"^([a-z_]+\.?)+$") # match key.attrib
if query and type(query) is bytes:
if query.find("|") != -1:
split_results = query.split("|")
query = split_results[0]
for item in split_results[1:]:
func = item.split("(")[0].strip()
for filter in self.filters:
if item.find(filter) == -1:
continue
for elem in item.replace(filter, '').split(','):
dasfilter = elem.strip()
if not dasfilter:
continue
if not pat.match(dasfilter):
msg = 'Incorrect filter: %s' % dasfilter
raise Exception(msg)
if dasfilter not in filters:
filters.append(dasfilter)
if func in self.aggregators:
aggregators = [agg for agg in get_aggregator(item)]
else:
mapreduce.append(item)
# mapreduce = [i.strip() for i in split_results[1:]]
query = query.strip()
if query[0] == "{" and query[-1] == "}":
mongo_query = json.loads(query)
if mongo_query.keys() != ['fields', 'spec']:
raise Exception("Invalid MongoDB query %s" % query)
if add_to_analytics:
self.analytics.add_query(query, mongo_query)
return mongo_query
# check input query and prepare it for processing
findbracketobj(query) # check brackets in a query
skeys = []
query = query.strip().replace(",", " ")
query = add_spaces(query, operators)
slist = query.split()
idx = 0
# main loop, step over words in query expression and
# findout selection keys and conditions
condlist = []
while True:
if idx >= len(slist):
break
word = slist[idx].strip()
if word in self.daskeys: # look-up for selection keys
try:
next_word = slist[idx + 1]
if next_word not in operators and word not in skeys:
skeys.append(word)
except:
pass
if word == slist[-1] and word not in skeys: # last word
skeys.append(word)
elif word in operators: # look-up conditions
oper = word
prev_word = slist[idx - 1]
next_word = slist[idx + 1]
if word in ['in', 'nin']:
first = next_word
if first.find('[') == -1:
msg = 'No open bracket [ found in query expression'
raise Exception(msg)
arr = []
found_last = False
for item in slist[idx + 1:]:
if item.find(']') != -1:
found_last = True
val = item.replace('[', '').replace(']', '')
if val:
arr.append(val)
if not found_last:
msg = 'No closed bracket ] found in query expression'
raise Exception(msg)
value = arr
elif word == 'last':
value = convert2date(next_word)
cdict = dict(key='date', op='in', value=value)
condlist.append(cdict)
value = None
else:
value = next_word
if prev_word == 'date':
if word != 'last': # we already converted date
if type(value) is bytes:
value = [das_dateformat(value), time.time()]
elif type(value) is list:
try:
value1 = das_dateformat(value[0])
value2 = das_dateformat(value[1])
value = [value1, value2]
except:
msg = "Unable to parse %s" % value
raise Exception(msg)
cdict = dict(key='date', op='in', value=value)
condlist.append(cdict)
value = None
idx += 1
if not value:
continue
key = prev_word
value = adjust_value(value)
if key == 'date':
cdict = dict(key=key, op=oper, value=value)
condlist.append(cdict)
continue
for system in self.map.list_systems():
mapkey = self.map.find_mapkey(system, key)
if mapkey:
cdict = dict(key=mapkey, op=oper, value=value)
if cdict not in condlist:
condlist.append(cdict)
else:
if word not in skeys and word in self.daskeys:
skeys.append(word)
idx += 1
if not condlist and skeys: # e.g. --query="dataset"
for key in skeys:
for system, daskeys in self.map.daskeys().items():
if key in daskeys:
mapkey = self.map.find_mapkey(system, key)
cdict = dict(key=mapkey, op="=", value="*")
condlist.append(cdict)
break
# print "\n### condlist", condlist
spec = mongo_exp(condlist)
# print "### spec", spec
if skeys:
fields = skeys
else:
fields = None
mongo_query = dict(fields=fields, spec=spec)
# add mapreduce if it exists
if mapreduce:
mongo_query['mapreduce'] = mapreduce
if filters:
mongo_query['filters'] = filters
if aggregators:
mongo_query['aggregators'] = aggregators
if add_to_analytics:
self.analytics.add_query(query, mongo_query)
return mongo_query
