def test_yajl(self): """Test yajl wrapper""" myjson.MODULE = 'yajl' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) data = {'a':1, 'b':2} kwds = {'sort_keys':True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b':2, 'a':1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = 123 result1 = myjson.JSONEncoder(**kwds).encode(data) kwds = {'sort_keys':True} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}} result1 = myjson.JSONEncoder().encode(data) result2 = myjson.JSONDecoder().decode(result1) self.assertEqual(data, result2) data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}} result1 = myjson.dumps(data) result2 = myjson.loads(result1) self.assertEqual(data, result2) myjson.MODULE = self.module
def send_request(self, method, kwargs): "Send POST request to server with provided parameters" url = self.cachesrv uinput = getarg(kwargs, 'input', '') format = getarg(kwargs, 'format', '') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) skey = getarg(kwargs, 'sort', '') sdir = getarg(kwargs, 'dir', 'asc') params = {'query':uinput, 'idx':idx, 'limit':limit, 'skey':skey, 'order':sdir} if method == 'POST': path = '/rest/create' elif method == 'GET': path = '/rest/request' else: raise Exception('Unsupported method %s' % method) headers = {'Accept': 'application/json', 'Content-type': 'application/json'} try: data = urllib2_request(method, url+path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {'status':'fail', 'reason':traceback.format_exc()} return result
def mongo_query(self): """ Read only mongo query, generated on demand. """ system = self._mongo_query.get('system', []) filters = self._mongo_query.get('filters', {}) aggregators = self._mongo_query.get('aggregators', []) if not self._mongo_query: self._mongo_query = deepcopy(self.storage_query) for key, val in self._mongo_query.items(): if key not in ['fields', 'spec']: setattr(self, '_%s' % key, val) spec = {} for item in self._mongo_query.pop('spec'): val = json.loads(item['value']) if 'pattern' in item: val = re.compile(val) spec.update({item['key'] : val}) self._mongo_query['spec'] = spec # special case when user asks for all records fields = self._mongo_query.get('fields', None) if fields and fields == ['records']: self._mongo_query['fields'] = None spec = {} for key, val in self._mongo_query['spec'].items(): if key != 'records': spec[key] = val self._mongo_query = dict(fields=None, spec=spec) if filters: self._mongo_query.update({'filters':filters}) if aggregators: self._mongo_query.update({'aggregators':aggregators}) if system: self._mongo_query.update({'system':system}) return self._mongo_query
def send_request(self, method, kwargs): "Send POST request to server with provided parameters" url = self.cachesrv uinput = getarg(kwargs, 'input', '') format = getarg(kwargs, 'format', '') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) skey = getarg(kwargs, 'sort', '') sdir = getarg(kwargs, 'dir', 'asc') params = { 'query': uinput, 'idx': idx, 'limit': limit, 'skey': skey, 'order': sdir } if method == 'POST': path = '/rest/create' elif method == 'GET': path = '/rest/request' else: raise Exception('Unsupported method %s' % method) headers = { 'Accept': 'application/json', 'Content-type': 'application/json' } try: data = urllib2_request(method, url + path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {'status': 'fail', 'reason': traceback.format_exc()} return result
def get_file_run_lumis(url, api, args, verbose=0): "Helper function to deal with file,run,lumi requests" run_value = args.get("run_num", []) if isinstance(run_value, dict) and "$in" in run_value: runs = run_value["$in"] elif isinstance(run_value, list): runs = run_value else: if int_number_pattern.match(str(run_value)): runs = [run_value] elif run_value[0] == "[" and run_value[-1] == "]": runs = json.loads(run_value) else: runs = [] args.update({"runs": runs}) blk = args.get("block_name", None) if blk: # we don't need to look-up blocks blocks = [blk] else: blocks = dbs_find("block", url, args, verbose) if not blocks: return gen = file_run_lumis(url, blocks, runs, verbose) key = "file_run" if api.startswith("run_lumi"): key = "run" if api.startswith("file_lumi"): key = "file" if api.startswith("file_run_lumi"): key = "file_run" for row in process_lumis_with(key, gen): yield row
def process(gen): "Process generator from getdata" for row in gen: if "error" in row: error = row.get("error") reason = row.get("reason", "") print dastimestamp("DAS ERROR"), error, reason yield row continue if "data" in row: yield json.loads(row["data"])
def process(gen): "Process generator from getdata" for row in gen: if 'error' in row: error = row.get('error') reason = row.get('reason', '') print(dastimestamp('DAS ERROR'), error, reason) yield row continue if 'data' in row: yield json.loads(row['data'])
def decode_mongo_query(query): """ Decode query from storage format into mongo format. """ spec = {} for item in query.pop('spec'): val = json.loads(item['value']) if 'pattern' in item: val = re.compile(val) spec.update({item['key'] : val}) query['spec'] = spec return query
def test_cjson(self): """Test cjson wrapper""" myjson.MODULE = 'cjson' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) myjson.MODULE = self.module
def get_all_views(self, dbname=None): """ Method to get all degined views in couch db. The couch db doesn't have a clear way to extract view documents. Instead we need to ask for _all_docs and provide proper start/end-keys. Once we retrieve _design docs, we loop over them and get the doc of particular view, e.g http://localhost:5984/das/_design/dasviews """ if not dbname: dbname = self.dbname qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22' host = 'http://' + self.uri path = '/%s/_all_docs?%s' % (dbname, qqq) kwds = {} req = 'GET' debug = 0 results = httplib_request(host, path, kwds, req, debug) designdocs = json.loads(results) results = {} for item in designdocs['rows']: doc = item['key'] # print "design:", doc path = '/%s/%s' % (dbname, doc) res = httplib_request(host, path, kwds, req, debug) rdict = json.loads(res) views = [] for view_name, view_dict in rdict['views'].items(): # print " view:", view_name # print " map:", view_dict['map'] if 'reduce' in view_dict: # print "reduce:", view_dict['reduce'] rdef = view_dict['reduce'] defrow = dict(map=view_dict['map'], reduce=view_dict['reduce']) else: defrow = dict(map=view_dict['map']) row = {'%s' % view_name: defrow} views.append(row) results[doc] = views return results
def get_all_views(self, dbname=None): """ Method to get all degined views in couch db. The couch db doesn't have a clear way to extract view documents. Instead we need to ask for _all_docs and provide proper start/end-keys. Once we retrieve _design docs, we loop over them and get the doc of particular view, e.g http://localhost:5984/das/_design/dasviews """ if not dbname: dbname = self.dbname qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22' host = 'http://' + self.uri path = '/%s/_all_docs?%s' % (dbname, qqq) kwds = {} req = 'GET' debug = 0 results = httplib_request(host, path, kwds, req, debug) designdocs = json.loads(results) results = {} for item in designdocs['rows']: doc = item['key'] # print "design:", doc path = '/%s/%s' % (dbname, doc) res = httplib_request(host, path, kwds, req, debug) rdict = json.loads(res) views = [] for view_name, view_dict in rdict['views'].items(): # print " view:", view_name # print " map:", view_dict['map'] if 'reduce' in view_dict: # print "reduce:", view_dict['reduce'] rdef = view_dict['reduce'] defrow = dict(map=view_dict['map'], reduce=view_dict['reduce']) else: defrow = dict(map=view_dict['map']) row = {'%s' % view_name : defrow} views.append(row) results[doc] = views return results
def configs(url, args, verbose=False): """Find config info in ReqMgr""" headers = {'Accept': 'application/json;text/json'} dataset = args.get('dataset', None) if not dataset: return base = 'https://%s' % url.split('/')[2] idict, source = findReqMgrIds(dataset, base, verbose) ids = [] ids_types = {} # keep track of ids/types relationship for key, ilist in idict.items(): rtype = 'output' if key.lower().find('output') != -1 else 'input' for item in ilist: ids.append(item) ids_types[item] = rtype # for hash ids find configs via ReqMgr REST API urls = [rurl(base, i) for i in ids if len(i) == 32] # for non-hash ids probe to find configs in showWorkload req_urls = ['%s/couchdb/reqmgr_workload_cache/%s' \ % (base, i) for i in ids if len(i) != 32] if req_urls: gen = urlfetch_getdata(req_urls, CKEY, CERT, headers) config_urls = [] for row in gen: if 'error' not in row: url = row['url'] for key, rtype in ids_types.items(): if key in url: break rdict = json.loads(row['data']) for key in rdict.keys(): val = rdict[key] if key.endswith('ConfigCacheID'): if isinstance(val, basestring): config_urls.append(rurl(base, val)) ids_types[val] = rtype elif isinstance(val, dict): for kkk in val.keys(): if kkk.endswith('ConfigCacheID'): vvv = val[kkk] if isinstance(vvv, basestring): config_urls.append(rurl(base, vvv)) ids_types[vvv] = rtype if config_urls: urls += config_urls udict = {} for rid, rtype in ids_types.items(): for url in set(urls): if rid in url: udict.setdefault(rtype, []).append(url) config = {'dataset':dataset, 'name': source, 'urls': udict, 'ids': ids, 'idict': idict} yield {'config': config}
def _test_yajl(self): """Test yajl wrapper""" import yajl myjson.MODULE = 'yajl' myjson.yajl = yajl # so not to depend on import sequence expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) data = {'a': 1, 'b': 2} kwds = {'sort_keys': True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b': 2, 'a': 1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = 123 result1 = myjson.JSONEncoder(**kwds).encode(data) kwds = {'sort_keys': True} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}} result1 = myjson.JSONEncoder().encode(data) result2 = myjson.JSONDecoder().decode(result1) self.assertEqual(data, result2) data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}} result1 = myjson.dumps(data) result2 = myjson.loads(result1) self.assertEqual(data, result2) myjson.MODULE = self.module
def status(self, **kwargs): """ Place request to obtain status about given query """ img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base req = """ <script type="application/javascript"> setTimeout('ajaxStatus()',3000) </script>""" def set_header(): "Set HTTP header parameters" tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) cherrypy.response.headers['Expire'] = tstamp cherrypy.response.headers['Cache-control'] = 'no-cache' uinput = kwargs.get('input', '') uinput = urllib.unquote_plus(uinput) ajax = kwargs.get('ajax', 1) view = kwargs.get('view', 'list') params = {'query':uinput} path = '/rest/status' url = self.cachesrv headers = {'Accept': 'application/json'} try: res = urllib2_request('GET', url+path, params, headers=headers) data = json.loads(res) except: self.daslogger.error(traceback.format_exc()) data = {'status':'fail'} if ajax: cherrypy.response.headers['Content-Type'] = 'text/xml' if data['status'] == 'ok': page = '<script type="application/javascript">reload()</script>' elif data['status'] == 'fail': page = '<script type="application/javascript">reload()</script>' page += self.error(self.gen_error_msg(kwargs)) else: page = img + ' ' + str(data['status']) + ', please wait...' img_stop = '' page += ', <a href="/das/">stop</a> request' page += req set_header() page = ajax_response(page) else: try: page = data['status'] except: page = traceback.format_exc() return page
def result(self, kwargs): """ invoke DAS search call, parse results and return them to web methods """ result = self.send_request("GET", kwargs) res = [] if type(result) is types.StringType: data = json.loads(result) else: data = result if data["status"] == "success": res = data["data"] return res
def status(self, **kwargs): """ Place request to obtain status about given query """ img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base req = """ <script type="application/javascript"> setTimeout('ajaxStatus()',3000) </script>""" def set_header(): "Set HTTP header parameters" tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) cherrypy.response.headers["Expire"] = tstamp cherrypy.response.headers["Cache-control"] = "no-cache" uinput = kwargs.get("input", "") uinput = urllib.unquote_plus(uinput) ajax = kwargs.get("ajax", 1) view = kwargs.get("view", "list") params = {"query": uinput} path = "/rest/status" url = self.cachesrv headers = {"Accept": "application/json"} try: res = urllib2_request("GET", url + path, params, headers=headers) data = json.loads(res) except: self.daslogger.error(traceback.format_exc()) data = {"status": "fail"} if ajax: cherrypy.response.headers["Content-Type"] = "text/xml" if data["status"] == "ok": page = '<script type="application/javascript">reload()</script>' elif data["status"] == "fail": page = '<script type="application/javascript">reload()</script>' page += self.error(self.gen_error_msg(kwargs)) else: page = img + " " + str(data["status"]) + ", please wait..." img_stop = "" page += ', <a href="/das/">stop</a> request' page += req set_header() page = ajax_response(page) else: try: page = data["status"] except: page = traceback.format_exc() return page
def status(self, **kwargs): """ Place request to obtain status about given query """ img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base req = """ <script type="application/javascript"> setTimeout('ajaxStatus()',3000) </script>""" def set_header(): "Set HTTP header parameters" tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) cherrypy.response.headers['Expire'] = tstamp cherrypy.response.headers['Cache-control'] = 'no-cache' uinput = kwargs.get('input', '') uinput = urllib.unquote_plus(uinput) ajax = kwargs.get('ajax', 1) view = kwargs.get('view', 'list') params = {'query': uinput} path = '/rest/status' url = self.cachesrv headers = {'Accept': 'application/json'} try: res = urllib2_request('GET', url + path, params, headers=headers) data = json.loads(res) except: self.daslogger.error(traceback.format_exc()) data = {'status': 'fail'} if ajax: cherrypy.response.headers['Content-Type'] = 'text/xml' if data['status'] == 'ok': page = '<script type="application/javascript">reload()</script>' elif data['status'] == 'fail': page = '<script type="application/javascript">reload()</script>' page += self.error(self.gen_error_msg(kwargs)) else: page = img + ' ' + str(data['status']) + ', please wait...' img_stop = '' page += ', <a href="/das/">stop</a> request' page += req set_header() page = ajax_response(page) else: try: page = data['status'] except: page = traceback.format_exc() return page
def result(self, kwargs): """ invoke DAS search call, parse results and return them to web methods """ result = self.send_request('GET', kwargs) res = [] if type(result) is bytes: data = json.loads(result) else: data = result if data['status'] == 'success': res = data['data'] return res
def phedex_info(urls, dbsdata): "Get phedex info for given set of dbs data" # create list of URLs for urlfetch url = urls.get('phedex') + '/blockReplicas' urls = ('%s?dataset=%s' % (url, d) for d in dbsdata.keys()) headers = {'Accept':'application/json;text/json'} gen = urlfetch_getdata(urls, CKEY, CERT, headers) for ddict in gen: try: jsondict = json.loads(ddict['data']) except Exception as _exc: continue rec = {} for blk in jsondict['phedex']['block']: dataset = blk['name'].split('#')[0] if 'nfiles' not in rec: nfiles = blk['files'] size = blk['bytes'] else: nfiles = rec['nfiles'] + blk['files'] size = rec['size'] + blk['bytes'] rec.update({'nfiles':nfiles, 'size':size}) for rep in blk['replica']: if 'site' not in rec: rec = dict(dataset=dataset, nfiles=nfiles, size=size, site=[rep['node']], se=[rep['se']], custodial=[rep['custodial']]) rec.update(dbsdata[dataset]) else: sites = rec['site'] ses = rec['se'] custodial = rec['custodial'] if rep['node'] not in sites: sites.append(rep['node']) ses.append(rep['se']) custodial.append(rep['custodial']) rec.update({'site':sites, 'se':ses, 'custodial':custodial}) if rec: # unwrap the site/se/custodial lists and yield records w/ their # individual values for idx in range(0, len(rec['site'])): sename = rec['se'][idx] site = rec['site'][idx] custodial = rec['custodial'][idx] newrec = dict(rec) newrec['se'] = sename newrec['site'] = site newrec['custodial'] = custodial yield newrec
def _test_cjson(self): """ Test cjson wrapper """ import cjson myjson.MODULE = 'cjson' myjson.cjson = cjson expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) myjson.MODULE = self.module
def block_summary(dbs, blocks): "Get block summary information for given set of blocks" headers = {"Accept": "text/json;application/json"} url = dbs + "/blocksummaries" urls = ["%s/?block_name=%s" % (url, urllib.quote(b)) for b in blocks] res = urlfetch_getdata(urls, CKEY, CERT, headers) for row in res: if "error" in row: error = row.get("error") reason = row.get("reason", "") yield {"error": error, "reason": reason} continue url = row["url"] blk = urllib.unquote(url.split("=")[-1]) for rec in json.loads(row["data"]): data = {"name": blk, "size": rec["file_size"], "nfiles": rec["num_file"], "nevents": rec["num_event"]} yield dict(block=data)
def test_json(self): """Test json wrapper""" myjson.MODULE = 'json' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) data = {'a':1, 'b':2} kwds = {'sort_keys':True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b':2, 'a':1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) myjson.MODULE = self.module
def records(self, *args, **kwargs): """ HTTP GET request. Retrieve records from provided collection. """ data = {'server_method': 'request'} if 'query' not in kwargs: data['status'] = 'fail' data['reason'] = 'no query is provided' return data # input query in JSON format, we should decode it using json. query = json.loads(kwargs.get('query')) coll = kwargs.get('collection', 'merge') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion count = kwargs.get('count', 0) data.update({ 'status': 'requested', 'query': kwargs['query'], 'collection': coll, 'count': count }) if '_id' in query['spec']: recid = query['spec']['_id'] ids = [] if type(recid) is bytes: ids = [ObjectId(recid)] elif type(recid) is list: ids = [ObjectId(r) for r in recid] spec = {'spec': {'_id': {'$in': ids}}} else: # look-up all records spec = {} self.logdb(query) try: gen = self.dascore.rawcache.get_from_cache\ (spec, idx=idx, limit=limit, collection=coll, adjust=False) data['status'] = 'success' data['data'] = [r for r in gen] except: self.debug(traceback.format_exc()) data['status'] = 'fail' data['reason'] = sys.exc_info()[0] return data
def block_summary(dbs, blocks): "Get block summary information for given set of blocks" headers = {'Accept':'text/json;application/json'} url = dbs + "/blocksummaries" urls = ['%s/?block_name=%s' % (url, urllib.quote(b)) for b in blocks] res = urlfetch_getdata(urls, CKEY, CERT, headers) for row in res: if 'error' in row: error = row.get('error') reason = row.get('reason', '') yield {'error':error, 'reason':reason} continue url = row['url'] blk = urllib.unquote(url.split('=')[-1]) for rec in json.loads(row['data']): data = {'name': blk, 'size': rec['file_size'], 'nfiles': rec['num_file'], 'nevents': rec['num_event']} yield dict(block=data)
def _test_json(self): """ Test json wrapper """ myjson.MODULE = 'json' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) data = {'a': 1, 'b': 2} kwds = {'sort_keys': True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b': 2, 'a': 1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) myjson.MODULE = self.module
def extract_http_error(err): """ Upon urllib failure the data-service can send HTTPError message. It can be in a form of JSON, etc. This function attempts to extract such message. If it fails it just str(err) and return. """ msg = str(err) try: err = json.loads(err) if err.has_key('message'): value = err['message'] if isinstance(value, dict): msg = '' for key, val in value.iteritems(): msg += '%s: %s. ' % (key, val) else: msg = str(value) except: pass return msg
def worker_v2(url, query): """ Query RunRegistry service, see documentation at https://twiki.cern.ch/twiki/bin/viewauth/CMS/DqmRrApi url=http://pccmsdqm04.cern.ch/runregistry/xmlrpc """ server = xmlrpclib.ServerProxy(url) namespace = 'GLOBAL' if isinstance(query, str) or isinstance(query, unicode): try: data = server.RunLumiSectionRangeTable.exportJson(namespace, query) except Exception as _err: data = "{}" # empty response for row in json.loads(data): yield row elif isinstance(query, dict): iformat = 'tsv_runs' # other formats are xml_all, csv_runs try: data = server.RunDatasetTable.export(namespace, iformat, query) except Exception as _err: data = "" # empty response titles = [] for line in data.split('\n'): if not line: continue if not titles: for title in line.split('\t')[:-1]: title = title.lower() if title != 'run_number': title = title.replace('run_', '') titles.append(title) continue val = line.split('\t')[:-1] if len(val) != len(titles): continue record = {} for idx in range(0, len(titles)): key = titles[idx] record[key] = adjust_value(val[idx]) yield dict(run=record)
def rest(self, *args, **kwargs): """ RESTful interface. We use args tuple as access method(s), e.g. args = ('method',) and kwargs to represent input parameters. """ request = cherrypy.request.method if request not in self.methods.keys(): msg = "Usupported request '%s'" % requset return {'error': msg} method = args[0] if method not in self.methods[request].keys(): msg = "Unsupported method '%s'" % method return {'error': msg} if request == 'POST': if cherrypy.request.body: body = cherrypy.request.body.read() try: kwargs = json.loads(body) except: msg = "Unable to load body request" return {'error': msg} return getattr(self, method)(kwargs)
def nresults(self, kwargs): """ invoke DAS search call, parse results and return them to web methods """ url = self.cachesrv uinput = getarg(kwargs, 'input', '') params = {'query': uinput} path = '/rest/nresults' headers = {"Accept": "application/json"} try: data = urllib2_request('GET', url + path, params, headers=headers) record = json.loads(data) except: self.daslogger.error(traceback.format_exc()) record = {'status': 'fail', 'reason': traceback.format_exc()} if record['status'] == 'success': return record['nresults'] else: msg = "nresults returns status: %s" % str(record) self.daslogger.info(msg) return -1
def nresults(self, kwargs): """ invoke DAS search call, parse results and return them to web methods """ url = self.cachesrv uinput = getarg(kwargs, "input", "") params = {"query": uinput} path = "/rest/nresults" headers = {"Accept": "application/json"} try: data = urllib2_request("GET", url + path, params, headers=headers) record = json.loads(data) except: self.daslogger.error(traceback.format_exc()) record = {"status": "fail", "reason": traceback.format_exc()} if record["status"] == "success": return record["nresults"] else: msg = "nresults returns status: %s" % str(record) self.daslogger.info(msg) return -1
def nresults(self, kwargs): """ invoke DAS search call, parse results and return them to web methods """ url = self.cachesrv uinput = getarg(kwargs, 'input', '') params = {'query':uinput} path = '/rest/nresults' headers = {"Accept": "application/json"} try: data = urllib2_request('GET', url+path, params, headers=headers) record = json.loads(data) except: self.daslogger.error(traceback.format_exc()) record = {'status':'fail', 'reason':traceback.format_exc()} if record['status'] == 'success': return record['nresults'] else: msg = "nresults returns status: %s" % str(record) self.daslogger.info(msg) return -1
def records(self, *args, **kwargs): """ HTTP GET request. Retrieve records from provided collection. """ data = {'server_method':'request'} if not kwargs.has_key('query'): data['status'] = 'fail' data['reason'] = 'no query is provided' return data # input query in JSON format, we should decode it using json. query = json.loads(kwargs.get('query')) coll = kwargs.get('collection', 'merge') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion count = kwargs.get('count', 0) data.update({'status':'requested', 'query':kwargs['query'], 'collection':coll, 'count': count}) if query['spec'].has_key('_id'): recid = query['spec']['_id'] ids = [] if type(recid) is types.StringType: ids = [ObjectId(recid)] elif type(recid) is types.ListType: ids = [ObjectId(r) for r in recid] spec = {'spec':{'_id':{'$in':ids}}} else: # look-up all records spec = {} self.logdb(query) try: gen = self.dascore.rawcache.get_from_cache\ (spec, idx=idx, limit=limit, collection=coll, adjust=False) data['status'] = 'success' data['data'] = [r for r in gen] except: self.debug(traceback.format_exc()) data['status'] = 'fail' data['reason'] = sys.exc_type return data
def get_file_run_lumis(url, api, args, verbose=0): "Helper function to deal with file,run,lumi requests" run_value = args.get('run_num', []) if isinstance(run_value, dict) and '$in' in run_value: runs = run_value['$in'] elif isinstance(run_value, list): runs = run_value else: if int_number_pattern.match(str(run_value)): runs = [run_value] elif run_value[0]=='[' and run_value[-1]==']': if '-' in run_value: # continuous range runs = run_value.replace("'", '').replace('[', '').replace(']', '') else: runs = json.loads(run_value) else: runs = run_value args.update({'runs': runs}) blk = args.get('block_name', None) if blk: # we don't need to look-up blocks blocks = [blk] else: blocks = dbs_find('block', url, args, verbose) if not blocks: return valid = 1 if args.get('validFileOnly', '') else 0 gen = file_run_lumis(url, blocks, runs, valid, verbose) key = 'file_run' if api.startswith('run_lumi'): key = 'run' if api.startswith('file_lumi'): key = 'file' if api.startswith('file_run'): key = 'file_run' if api.startswith('file_run_lumi'): key = 'file_run' for row in process_lumis_with(key, gen): yield row
def parser(self, dasquery, dformat, source, args): """ Data parser for Monitor service. """ if isinstance(source, InstanceType) or isinstance(source, file): try: # we got data descriptor data = source.read() except: source.close() raise source.close() elif isinstance(source, object) and hasattr(source, 'read'): # StringIO data = source.read() source.close() else: data = source try: row = json.loads(data) except: msg = "MonitorService::parser," msg += " WARNING, fail to JSON'ify data:\n%s" % data self.logger.warning(msg) row = eval(data, { "__builtins__": None }, {}) try: monitor_time = row['series'] monitor_data = row['data'] items = ({'time':list(t), 'data':d} for t, d in \ zip(monitor_time, monitor_data)) for row in items: interval = row['time'] dataval = row['data'] for key, val in dataval.iteritems(): newrow = {'time': interval} newrow[args['grouping']] = key newrow['rate'] = val yield dict(monitor=newrow) except: yield dict(monitor=row)
def file_run_lumis(url, blocks, runs=None, valid=None, verbose=0): """ Find file, run, lumi tuple for given set of files and (optional) runs. """ headers = {'Accept': 'application/json;text/json'} urls = [] for blk in blocks: if not blk: continue dbs_url = '%s/filelumis/?block_name=%s' % (url, urllib.quote(blk)) if valid: dbs_url += '&validFileOnly=1' if runs: dbs_url += "&run_num=%s" % urllib.quote(str(runs)) urls.append(dbs_url) if not urls: return if verbose > 1: print("\nDEBUG: file_run_lumis") print(urls) gen = urlfetch_getdata(urls, CKEY, CERT, headers) odict = {} # output dict for rec in gen: if 'error' in rec: error = rec.get('error') reason = rec.get('reason', '') yield {'error':error, 'reason':reason} else: for row in json.loads(rec['data']): run = row['run_num'] lfn = row['logical_file_name'] lumilist = row['lumi_section_num'] key = (lfn, run) for lumi in lumilist: odict.setdefault(key, []).append(lumi) for key, lumis in odict.items(): lfn, run = key yield lfn, run, lumis
def urlfetch_proxy(urls): "Proxy client for Go proxy server" params = {'urls': '\n'.join(urls)} encoded_data = urllib.urlencode(params) server = "http://localhost:8215/fetch" req = urllib2.Request(server) data = urllib2.urlopen(req, encoded_data) code = data.getcode() if code == 200: if not urls: # ping request yield {'ping':'pong'} else: for row in data.readlines(): if row and row[0] == '{' and row.find('data'): # Erlang response rec = json.loads(row) for line in rec['data'].split('\n'): yield line else: # Go response yield row else: yield {'error':'Fail to contact UrlFetch proxy server', 'code':code}
def urlfetch_proxy(urls): "Proxy client for Go proxy server" params = {"urls": "\n".join(urls)} encoded_data = urllib.urlencode(params) server = "http://localhost:8215/fetch" req = urllib2.Request(server) data = urllib2.urlopen(req, encoded_data) code = data.getcode() if code == 200: if not urls: # ping request yield {"ping": "pong"} else: for row in data.readlines(): if row and row[0] == "{" and row.find("data"): # Erlang response rec = json.loads(row) for line in rec["data"].split("\n"): yield line else: # Go response yield row else: yield {"error": "Fail to contact UrlFetch proxy server", "code": code}
def block_run_lumis(url, blocks, runs=None, verbose=0): """ Find block, run, lumi tuple for given set of files and (optional) runs. """ headers = {"Accept": "application/json;text/json"} urls = [] params = {} for blk in blocks: if not blk: continue dbs_url = "%s/filelumis/?block_name=%s" % (url, urllib.quote(blk)) if runs and isinstance(runs, list): params.update({"run_num": runrange(runs[0], runs[-1], False)}) urls.append(dbs_url) if not urls: return if verbose > 1: print "\nDEBUG: block_run_lumis" print urls gen = urlfetch_getdata(urls, CKEY, CERT, headers) odict = {} # output dict for rec in gen: blk = urllib.unquote(url_args(rec["url"])["block_name"]) if "error" in rec: error = rec.get("error") reason = rec.get("reason", "") yield {"error": error, "reason": reason} else: for row in json.loads(rec["data"]): run = row["run_num"] lumilist = row["lumi_section_num"] key = (blk, run) for lumi in lumilist: odict.setdefault(key, []).append(lumi) for key, lumis in odict.iteritems(): blk, run = key yield blk, run, lumis
def wrapper(self, *args, **kwds): """Wrapper for decorator""" # check request headers. For methods POST/PUT # we need to read request body to get parameters headers = cherrypy.request.headers if cherrypy.request.method == 'POST' or\ cherrypy.request.method == 'PUT': body = cherrypy.request.body.read() if args and kwds: msg = 'Misleading request.\n' msg += 'Request: %s\n' % cherrypy.request.method msg += 'Headers: %s\n' % headers msg += 'Parameters: args=%s, kwds=%s\n' % (args, kwds) return {'status': 'fail', 'reason': msg} if body: jsondict = json.loads(body, encoding='latin-1') else: jsondict = kwds for key, val in jsondict.items(): kwds[str(key)] = str(val) # headers = cherrypy.request.headers # if headers.has_key('Content-type'): # content = headers['Content-type'] # cherrypy.response.headers['Content-type'] = content # if content in ['application/json', 'text/json', 'text/x-json']: # body = cherrypy.request.body.read() # if args and kwds: # msg = 'Misleading request.' # msg += 'Headers: %s ' % headers # msg += 'Parameters: %s, %s' % (args, kwds) # return {'status':'fail', 'reason': msg} # jsondict = json.loads(body, encoding='latin-1') # for key, val in jsondict.items(): # kwds[str(key)] = str(val) pat = re.compile('^[+]?\d*$') supported = [ 'query', 'idx', 'limit', 'expire', 'method', 'skey', 'order', 'collection' ] if not kwds: if args: kwds = args[-1] keys = [] if kwds: keys = [i for i in kwds.keys() if i not in supported] if keys: msg = 'Unsupported keys: %s' % keys return {'status': 'fail', 'reason': msg} if 'idx' in kwds and not pat.match(str(kwds['idx'])): msg = 'Unsupported value idx=%s' % (kwds['idx']) return {'status': 'fail', 'reason': msg} if 'limit' in kwds and not pat.match(str(kwds['limit'])): msg = 'Unsupported value limit=%s' % (kwds['limit']) return {'status': 'fail', 'reason': msg} if 'expire' in kwds and not pat.match(str(kwds['expire'])): msg = 'Unsupported value expire=%s' % (kwds['expire']) return {'status': 'fail', 'reason': msg} if 'order' in kwds: if kwds['order'] not in ['asc', 'desc']: msg = 'Unsupported value order=%s' % (kwds['order']) return {'status': 'fail', 'reason': msg} data = func(self, *args, **kwds) return data
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None format = '' if args: recordid = args[0] spec = {'_id': recordid} fields = None query = dict(fields=fields, spec=spec) if len(args) == 2: format = args[1] elif kwargs and '_id' in kwargs: spec = {'_id': kwargs['_id']} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) nresults = self.nresults(query) time0 = time.time() url = self.cachesrv idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) show = getarg(kwargs, 'show', 'json') coll = getarg(kwargs, 'collection', 'merge') # params = {'query':json.dumps(query), 'idx':idx, 'limit':limit} # path = '/rest/request' params = { 'query': json.dumps(query), 'idx': idx, 'limit': limit, 'collection': coll } path = '/rest/records' headers = {"Accept": "application/json"} try: data = urllib2_request('GET', url + path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {'status': 'fail', 'reason': traceback.format_exc()} res = "" if result['status'] == 'success': if recordid: # we got id for row in result['data']: if show == 'json': jsoncode = {'jsoncode': json2html(row, "")} res += self.templatepage('das_json', **jsoncode) elif show == 'code': code = pformat(row, indent=1, width=100) res += self.templatepage('das_code', code=code) else: code = yaml.dump(row, width=100, indent=4, default_flow_style=False) res += self.templatepage('das_code', code=code) else: for row in result['data']: rid = row['_id'] del row['_id'] record = dict(id=rid, daskeys=', '.join(row)) res += self.templatepage('das_record', **record) else: res = result['status'] if 'reason' in res: return self.error(res['reason']) else: msg = 'Uknown error, kwargs=' % kwargs return self.error(msg) if recordid: if format: if format == 'xml': return self.wrap2dasxml(result['data']) elif format == 'json': return self.wrap2dasjson(result['data']) else: return self.error('Unsupported data format %s' % format) page = res else: url = '/das/records?' idict = dict(nrows=nresults, idx=idx, limit=limit, results=res, url=url) page = self.templatepage('das_pagination', **idict) form = self.form(uinput="") ctime = (time.time() - time0) page = self.page(form + page, ctime=ctime) return page except: return self.error(self.gen_error_msg(kwargs))
def requestquery(self, query, add_to_analytics=True): """ Query analyzer which form request query to DAS from a free text-based form. Return MongoDB request query. """ # strip operators while we will match words against them operators = [o.strip() for o in self.operators] # find out if input query contains filters/mapreduce functions mapreduce = [] filters = [] aggregators = [] pat = re.compile(r"^([a-z_]+\.?)+$") # match key.attrib if query and type(query) is bytes: if query.find("|") != -1: split_results = query.split("|") query = split_results[0] for item in split_results[1:]: func = item.split("(")[0].strip() for filter in self.filters: if item.find(filter) == -1: continue for elem in item.replace(filter, '').split(','): dasfilter = elem.strip() if not dasfilter: continue if not pat.match(dasfilter): msg = 'Incorrect filter: %s' % dasfilter raise Exception(msg) if dasfilter not in filters: filters.append(dasfilter) if func in self.aggregators: aggregators = [agg for agg in get_aggregator(item)] else: mapreduce.append(item) # mapreduce = [i.strip() for i in split_results[1:]] query = query.strip() if query[0] == "{" and query[-1] == "}": mongo_query = json.loads(query) if mongo_query.keys() != ['fields', 'spec']: raise Exception("Invalid MongoDB query %s" % query) if add_to_analytics: self.analytics.add_query(query, mongo_query) return mongo_query # check input query and prepare it for processing findbracketobj(query) # check brackets in a query skeys = [] query = query.strip().replace(",", " ") query = add_spaces(query, operators) slist = query.split() idx = 0 # main loop, step over words in query expression and # findout selection keys and conditions condlist = [] while True: if idx >= len(slist): break word = slist[idx].strip() if word in self.daskeys: # look-up for selection keys try: next_word = slist[idx + 1] if next_word not in operators and word not in skeys: skeys.append(word) except: pass if word == slist[-1] and word not in skeys: # last word skeys.append(word) elif word in operators: # look-up conditions oper = word prev_word = slist[idx - 1] next_word = slist[idx + 1] if word in ['in', 'nin']: first = next_word if first.find('[') == -1: msg = 'No open bracket [ found in query expression' raise Exception(msg) arr = [] found_last = False for item in slist[idx + 1:]: if item.find(']') != -1: found_last = True val = item.replace('[', '').replace(']', '') if val: arr.append(val) if not found_last: msg = 'No closed bracket ] found in query expression' raise Exception(msg) value = arr elif word == 'last': value = convert2date(next_word) cdict = dict(key='date', op='in', value=value) condlist.append(cdict) value = None else: value = next_word if prev_word == 'date': if word != 'last': # we already converted date if type(value) is bytes: value = [das_dateformat(value), time.time()] elif type(value) is list: try: value1 = das_dateformat(value[0]) value2 = das_dateformat(value[1]) value = [value1, value2] except: msg = "Unable to parse %s" % value raise Exception(msg) cdict = dict(key='date', op='in', value=value) condlist.append(cdict) value = None idx += 1 if not value: continue key = prev_word value = adjust_value(value) if key == 'date': cdict = dict(key=key, op=oper, value=value) condlist.append(cdict) continue for system in self.map.list_systems(): mapkey = self.map.find_mapkey(system, key) if mapkey: cdict = dict(key=mapkey, op=oper, value=value) if cdict not in condlist: condlist.append(cdict) else: if word not in skeys and word in self.daskeys: skeys.append(word) idx += 1 if not condlist and skeys: # e.g. --query="dataset" for key in skeys: for system, daskeys in self.map.daskeys().items(): if key in daskeys: mapkey = self.map.find_mapkey(system, key) cdict = dict(key=mapkey, op="=", value="*") condlist.append(cdict) break # print "\n### condlist", condlist spec = mongo_exp(condlist) # print "### spec", spec if skeys: fields = skeys else: fields = None mongo_query = dict(fields=fields, spec=spec) # add mapreduce if it exists if mapreduce: mongo_query['mapreduce'] = mapreduce if filters: mongo_query['filters'] = filters if aggregators: mongo_query['aggregators'] = aggregators if add_to_analytics: self.analytics.add_query(query, mongo_query) return mongo_query