def test_yajl(self): """Test yajl wrapper""" myjson.MODULE = 'yajl' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) data = {'a':1, 'b':2} kwds = {'sort_keys':True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b':2, 'a':1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = 123 result1 = myjson.JSONEncoder(**kwds).encode(data) kwds = {'sort_keys':True} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}} result1 = myjson.JSONEncoder().encode(data) result2 = myjson.JSONDecoder().decode(result1) self.assertEqual(data, result2) data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}} result1 = myjson.dumps(data) result2 = myjson.loads(result1) self.assertEqual(data, result2) myjson.MODULE = self.module
def encode_mongo_query(query): """ Encode mongo query into storage format. MongoDB does not allow storage of dict with keys containing "." or MongoDB operators, e.g. $lt. So we convert input mongo query spec into list of dicts whose "key"/"value" are mongo query spec key/values. For example .. doctest:: spec:{"block.name":"aaa"} converted into spec:[{"key":"block.name", "value":'"aaa"'}] Conversion is done using JSON dumps method. """ if not query: msg = 'Cannot decode, input query=%s' % query raise Exception(msg) return_query = dict(query) speclist = [] for key, val in return_query.pop('spec').items(): if str(type(val)) == "<type '_sre.SRE_Pattern'>": val = json.dumps(val.pattern) speclist.append({"key":key, "value":val, "pattern":1}) else: val = json.dumps(val) speclist.append({"key":key, "value":val}) return_query['spec'] = speclist return return_query
def keys(self, **kwds): """ Show DAS keys and their attibutes """ adict = {} for row in self.dasmgr.keylearning.attributes(): try: qpat = row.get('query_pat', []) key, attr = row['member'].split('.', 1) except: continue if key in adict: vdict = adict[key] if attr in vdict: vdict[attr] += qpat else: vdict[attr] = qpat adict[key] = vdict else: adict[key] = {attr: qpat} view = kwds.get('view', '') if view == 'json': return json.dumps(adict) page = self.templatepage('das_keys_attrs', attrs=adict) return self.page(page, response_div=False)
def das2api(self, system, api, rec_key, value=None): """ Translates DAS record key into data-service API input parameter, e.g. run.number => run_number """ entry = self.dasmapscache.get((system, api), None) names = [] if not entry: return [rec_key] for row in entry.get('das_map', []): if 'api_arg' in row: api_param = row['api_arg'] pat = row.get('pattern', None) if row['rec_key'] != rec_key: continue if value and pat: if isinstance(value, dict): if pat.match(json.dumps(value.values()[0])): if api_param not in names: names.append(api_param) if pat.match(str(value)): if api_param not in names: names.append(api_param) else: if api_param not in names: names.append(api_param) else: names.append(row['rec_key']) return names
def form(self, uinput="", instance=None, view="list", cards=False): """ provide input DAS search form """ # TODO: rename into search_form()? (template is also called like this if "'" in uinput: # e.g. file.creation_date>'20120101 12:01:01' uinput = uinput.replace("'", '"') if not instance: instance = self.dbs_global cards = self.templatepage( "das_cards", base=self.base, show=cards, width=900, height=220, cards=help_cards(self.base) ) daskeys = self.templatepage("das_keys", daskeys=self.daskeyslist) page = self.templatepage( "das_searchform", input=uinput, init_dbses=list(self.dbs_instances), daskeys=daskeys, base=self.base, instance=instance, view=view, cards=cards, autocompl_host=json.dumps(self._get_autocompl_host()), ) return page
def worker_v3(url, query): """ Query RunRegistry service, see documentation at https://twiki.cern.ch/twiki/bin/viewauth/CMS/DqmRrApi url=http://runregistry.web.cern.ch/runregistry/ """ workspace = 'GLOBAL' table = 'runsummary' template = 'json' columns = ['number', 'startTime', 'stopTime', 'triggers', 'runClassName', 'runStopReason', 'bfield', 'gtKey', 'l1Menu', 'hltKeyDescription', 'lhcFill', 'lhcEnergy', 'runCreated', 'modified', 'lsCount', 'lsRanges'] sdata = json.dumps({'filter':query}) path = 'api/GLOBAL/%s/%s/%s/none/data' \ % (table, template, urllib.quote(','.join(columns))) callurl = os.path.join(url, path) result = urllib.urlopen(callurl, sdata) record = json.load(result) result.close() notations = {'lsRanges':'lumi_section_ranges', 'number':'run_number', 'runCreated':'create_time', 'stopTime': 'end_time', 'startTime': 'start_time', 'lsCount': 'lumi_sections', 'runStopReason': 'stop_reason', 'hltKeyDescription': 'hltkey', 'gtKey': 'gtkey', 'lhcEnergy': 'beam_e', 'l1Menu': 'l1key', 'modified': 'modify_time', 'runClassName': 'group_name'} for rec in record: for key, val in rec.items(): if notations.has_key(key): rec[notations[key]] = val del rec[key] yield dict(run=rec)
def storage_query(self): """ Read only storage query, generated on demand. """ if not self._storage_query: self._storage_query = deepcopy(self.mongo_query) speclist = [] for key, val in self._storage_query.pop('spec').items(): if str(type(val)) == "<type '_sre.SRE_Pattern'>": val = json.dumps(val.pattern) speclist.append({"key":key, "value":val, "pattern":1}) elif isinstance(val, ObjectId): speclist.append({"key":key, "value":str(val)}) else: val = json.dumps(val) speclist.append({"key":key, "value":val}) self._storage_query['spec'] = speclist return self._storage_query
def test_cjson(self): """Test cjson wrapper""" myjson.MODULE = 'cjson' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) myjson.MODULE = self.module
def dump(ilist, idx=0): """ Print items in provided generator """ if isinstance(ilist, GeneratorType): reslist = ilist elif not isinstance(ilist, list): reslist = [ilist] else: reslist = ilist if not reslist: print "dump, no results found" return idx = 0 for row in reslist: print "id : %s" % idx if isinstance(row, dict): print json.dumps(row) else: print row print idx += 1
def check_map_record(rec): "Check hash of given map record" # remove _id MongoDB Object if "_id" in rec: del rec["_id"] if "hash" in rec: md5 = rec.pop("hash") rec_md5 = md5hash(rec) if rec_md5 != md5: err = "Invalid hash record:\n%s\n" % json.dumps(rec) err += "\nrecord hash : %s" % md5 err += "\nobtained hash: %s\n" % md5hash(rec) raise Exception(err)
def _test_yajl(self): """Test yajl wrapper""" import yajl myjson.MODULE = 'yajl' myjson.yajl = yajl # so not to depend on import sequence expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) data = {'a': 1, 'b': 2} kwds = {'sort_keys': True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b': 2, 'a': 1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = 123 result1 = myjson.JSONEncoder(**kwds).encode(data) kwds = {'sort_keys': True} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}} result1 = myjson.JSONEncoder().encode(data) result2 = myjson.JSONDecoder().decode(result1) self.assertEqual(data, result2) data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}} result1 = myjson.dumps(data) result2 = myjson.loads(result1) self.assertEqual(data, result2) myjson.MODULE = self.module
def check_map_record(record): "Check hash of given map record" rec = dict(record) # remove _id MongoDB Object for key in ['_id']+TRANSIENT_FIELDS: if key in rec: del rec[key] if 'hash' in rec: md5 = rec.pop('hash') rec_md5 = md5hash(rec) if rec_md5 != md5: err = 'Invalid hash record:\n%s\n' % json.dumps(rec) err += '\nrecord hash : %s' % md5 err += '\nobtained hash: %s\n' % md5hash(rec) raise Exception(err)
def _test_cjson(self): """ Test cjson wrapper """ import cjson myjson.MODULE = 'cjson' myjson.cjson = cjson expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) myjson.MODULE = self.module
def getdata_pycurl( url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, ): "Fetch data via pycurl library" contact = "data-service." if system: contact = system + " " + contact timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = "urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} try: reason = extract_http_error(httperror.read()) data.update({"reason": reason, "request": msg}) msg += "\n" + err except Exception as exp: data.update({"httperror": None}) msg += "\n" + str(exp) print dastimestamp("getdata_pycurl"), msg data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = "HTTPError, system=%s, url=%s, args=%s, headers=%s" % ( system, url, json.dumps(params), json.dumps(headers), ) print dastimestamp("getdata_pycurl"), msg + "\n" + str(exp) data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def gridfs(self, **kwargs): """ Retieve records from GridFS """ time0 = time.time() if 'fid' not in kwargs: code = web_code('No file id') raise HTTPError(500, 'DAS error, code=%s' % code) fid = kwargs.get('fid') data = {'status':'requested', 'fid':fid} try: fds = self.gfs.get(ObjectId(fid)) return fds.read() except Exception as exc: print_exc(exc) code = web_code('Exception') raise HTTPError(500, 'DAS error, code=%s' % code) data['ctime'] = time.time() - time0 return json.dumps(data)
def gridfs(self, **kwargs): """ Retieve records from GridFS """ time0 = time.time() if "fid" not in kwargs: code = web_code("No file id") raise HTTPError(500, "DAS error, code=%s" % code) fid = kwargs.get("fid") data = {"status": "requested", "fid": fid} try: fds = self.gfs.get(ObjectId(fid)) return fds.read() except Exception as exc: print_exc(exc) code = web_code("Exception") raise HTTPError(500, "DAS error, code=%s" % code) data["ctime"] = time.time() - time0 return json.dumps(data)
def test_json(self): """Test json wrapper""" myjson.MODULE = 'json' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data)) self.assertEqual(expect, result) data = {'a':1, 'b':2} kwds = {'sort_keys':True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b':2, 'a':1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) myjson.MODULE = self.module
def _test_json(self): """ Test json wrapper """ myjson.MODULE = 'json' expect = self.data result = myjson.loads(myjson.dumps(self.data)) self.assertEqual(expect, result) expect = self.data result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode( self.data)) self.assertEqual(expect, result) data = {'a': 1, 'b': 2} kwds = {'sort_keys': True} result1 = myjson.JSONEncoder(**kwds).encode(data) data = {'b': 2, 'a': 1} result2 = myjson.JSONEncoder(**kwds).encode(data) self.assertEqual(result1, result2) myjson.MODULE = self.module
def urllib2_request(request, url, params, headers=None, debug=0): """request method using GET request from urllib2 library""" if not headers: headers = {} if request == "GET" or request == "DELETE": encoded_data = urllib.urlencode(params, doseq=True) url += "?%s" % encoded_data req = UrlRequest(request, url=url, headers=headers) else: encoded_data = json.dumps(params) req = UrlRequest(request, url=url, data=encoded_data, headers=headers) if debug: hdlr = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(hdlr) else: opener = urllib2.build_opener() fdesc = opener.open(req) result = fdesc.read() fdesc.close() return result
def blocks4tier_date(dbs, tier, min_cdate, max_cdate, verbose=0): "Get list of blocks for given parameters" headers = {"Accept": "text/json;application/json"} url = dbs + "/blocks" params = {"data_tier_name": tier, "min_cdate": min_cdate, "max_cdate": max_cdate} urls = ["%s?%s" % (url, urllib.urlencode(params))] if verbose > 1: print "\nblocks4tier_date" print urls res = process(urlfetch_getdata(urls, CKEY, CERT, headers)) err = "Unable to get blocks for tier=%s, mindate=%s, maxdate=%s" % (tier, min_cdate, max_cdate) for blist in res: if "error" in blist: yield blist continue if isinstance(blist, dict): if "block_name" not in blist: msg = err + ", reason=%s" % json.dumps(blist) raise Exception(msg) for row in blist: yield row["block_name"]
def form(self, uinput='', instance=None, view='list', cards=False): """ provide input DAS search form """ # TODO: rename into search_form()? (template is also called like this if "'" in uinput: # e.g. file.creation_date>'20120101 12:01:01' uinput = uinput.replace("'", '"') if not instance: instance = self.dbs_global hcards = help_cards(self.base) width = 900 height = 220 cards = self.templatepage('das_cards', base=self.base, show=cards, \ width=width, height=height, max_width=len(hcards)*width, \ cards=hcards, enumerate=enumerate) daskeys = self.templatepage('das_keys', daskeys=self.daskeyslist) page = self.templatepage('das_searchform', input=uinput, \ init_dbses=list(self.dbs_instances), daskeys=daskeys, \ base=self.base, instance=instance, view=view, cards=cards, autocompl_host=json.dumps(self._get_autocompl_host()) ) return page
def wrapper (self, *args, **kwds): """Decorator wrapper""" cherrypy.response.headers['Content-Type'] = "application/json" func._cp_config = {'response.stream': True} head, data = func (self, *args, **kwds) yield json.dumps(head)[:-1] # do not yield } yield ', "data": [' if isinstance(data, dict): for chunk in JSONEncoder().iterencode(data): yield chunk elif isinstance(data, list) or isinstance(data, types.GeneratorType): sep = '' for rec in data: if sep: yield sep for chunk in JSONEncoder().iterencode(rec): yield chunk if not sep: sep = ', ' else: msg = 'jsonstreamer, improper data type %s' % type(data) raise Exception(msg) yield ']}'
def add(self, pid, kwds): """Add new pid/kwds""" self.clean() if not kwds: return tstamp = time.strftime("%Y%m%d %H:%M:%S", time.localtime()) doc = dict(_id=pid, kwds=json.dumps(kwds), ts=time.time(), timestamp=tstamp) attempts = 0 while True: try: self.col.insert(doc, safe=True) break except DuplicateKeyError as err: break except Exception as err: print_exc(err) time.sleep(0.01) attempts += 1 if attempts > 2: msg = '%s unable to add pid=%s' % (self.col, pid) print dastimestamp('DAS ERROR '), msg break
def blocks4tier_date(dbs, tier, min_cdate, max_cdate, verbose=0): "Get list of blocks for given parameters" headers = {'Accept':'text/json;application/json'} url = dbs + "/blocks" params = {'data_tier_name':tier, 'min_cdate':min_cdate, 'max_cdate':max_cdate} urls = ['%s?%s' % (url, urllib.urlencode(params))] if verbose > 1: print("\nblocks4tier_date") print(urls) res = process(urlfetch_getdata(urls, CKEY, CERT, headers)) err = 'Unable to get blocks for tier=%s, mindate=%s, maxdate=%s' \ % (tier, min_cdate, max_cdate) for blist in res: if 'error' in blist: yield blist continue if isinstance(blist, dict): if 'block_name' not in blist: msg = err + ', reason=%s' % json.dumps(blist) raise Exception(msg) for row in blist: yield row['block_name']
def getdata_pycurl(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None): "Fetch data via pycurl library" contact = 'data-service.' if system: contact = system + ' ' + contact if isinstance(params, dict): timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) else: timer_key = '%s?%s' % (url, params) das_timer(timer_key, verbose) handler = REQUEST_HANDLER try: data, expire = handler.getdata(url, params, headers, expire, post, \ error_expire, verbose, ckey, cert, doseq) except urllib2.HTTPError as httperror: msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) data = {'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} try: reason = extract_http_error(httperror.read()) data.update({'reason': reason, 'request': msg}) # TODO: err variable did not exit in this function! msg += '\n' + reason except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(dastimestamp('getdata_pycurl'), msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \ % (system, url, json.dumps(params), json.dumps(headers)) print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp)) data = {'error': 'Received generic error from %s data-service' % contact, 'reason': msg, 'ts':time.time()} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None if args: recordid = args[0] spec = {"_id": ObjectId(recordid)} fields = None query = dict(fields=fields, spec=spec) elif kwargs and "_id" in kwargs: spec = {"_id": ObjectId(kwargs["_id"])} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) res = "" time0 = time.time() idx = getarg(kwargs, "idx", 0) limit = getarg(kwargs, "limit", 10) coll = kwargs.get("collection", "merge") view = kwargs.get("view", "") if view == "json": res = [] inst = kwargs.get("instance", self.dbs_global) form = self.form(uinput="") check, content = self.generate_dasquery(query, inst) if check: return self.page(form + content, ctime=time.time() - time0) dasquery = content # returned content is valid DAS query nresults = self.dasmgr.rawcache.nresults(dasquery, coll) gen = self.dasmgr.rawcache.get_from_cache(dasquery, idx=idx, limit=limit, collection=coll) if recordid: # we got id for row in gen: if view == "json": res.append(row) else: res += das_json(dasquery, row) else: for row in gen: rid = row["_id"] del row["_id"] res += self.templatepage("das_record", id=rid, collection=coll, daskeys=", ".join(row)) if recordid: page = res else: url = "/das/records?" if nresults: page = self.templatepage("das_pagination", nrows=nresults, idx=idx, limit=limit, url=url) else: page = "No results found, nresults=%s" % nresults page += res ctime = time.time() - time0 if view == "json": return json.dumps(res) page = self.page(form + page, ctime=ctime) return page except Exception as exc: print_exc(exc) return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None format = "" if args: recordid = args[0] spec = {"_id": recordid} fields = None query = dict(fields=fields, spec=spec) if len(args) == 2: format = args[1] elif kwargs and kwargs.has_key("_id"): spec = {"_id": kwargs["_id"]} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) nresults = self.nresults(query) time0 = time.time() url = self.cachesrv idx = getarg(kwargs, "idx", 0) limit = getarg(kwargs, "limit", 10) show = getarg(kwargs, "show", "json") coll = getarg(kwargs, "collection", "merge") # params = {'query':json.dumps(query), 'idx':idx, 'limit':limit} # path = '/rest/request' params = {"query": json.dumps(query), "idx": idx, "limit": limit, "collection": coll} path = "/rest/records" headers = {"Accept": "application/json"} try: data = urllib2_request("GET", url + path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {"status": "fail", "reason": traceback.format_exc()} res = "" if result["status"] == "success": if recordid: # we got id for row in result["data"]: if show == "json": jsoncode = {"jsoncode": json2html(row, "")} res += self.templatepage("das_json", **jsoncode) elif show == "code": code = pformat(row, indent=1, width=100) res += self.templatepage("das_code", code=code) else: code = yaml.dump(row, width=100, indent=4, default_flow_style=False) res += self.templatepage("das_code", code=code) else: for row in result["data"]: rid = row["_id"] del row["_id"] record = dict(id=rid, daskeys=", ".join(row)) res += self.templatepage("das_record", **record) else: res = result["status"] if res.has_key("reason"): return self.error(res["reason"]) else: msg = "Uknown error, kwargs=" % kwargs return self.error(msg) if recordid: if format: if format == "xml": return self.wrap2dasxml(result["data"]) elif format == "json": return self.wrap2dasjson(result["data"]) else: return self.error("Unsupported data format %s" % format) page = res else: url = "/das/records?" idict = dict(nrows=nresults, idx=idx, limit=limit, results=res, url=url) page = self.templatepage("das_pagination", **idict) form = self.form(uinput="") ctime = time.time() - time0 page = self.page(form + page, ctime=ctime) return page except: return self.error(self.gen_error_msg(kwargs))
def getdata_urllib( url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, ): """ Invoke URL call and retrieve data from data-service based on provided URL and set of parameters. Use post=True to invoke POST request. """ contact = "data-service." if system: contact = system + " " + contact timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) encoded_data = urllib.urlencode(params, doseq=doseq) if not post: url = url + "?" + encoded_data if not headers: headers = {} if verbose: print "+++ getdata, url=%s, headers=%s" % (url, headers) req = urllib2.Request(url) for key, val in headers.iteritems(): req.add_header(key, val) if verbose > 1: handler = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) if ckey and cert: handler = HTTPSClientAuthHandler(ckey, cert, verbose) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) try: time0 = time.time() if post: data = urllib2.urlopen(req, encoded_data) else: data = urllib2.urlopen(req) data_srv_time = time.time() - time0 try: # get HTTP header and look for Expires e_time = expire_timestamp(data.info().__dict__["dict"]["expires"]) if e_time < expire_timestamp(data_srv_time): expire = max(e_time, expire_timestamp(expire)) elif e_time > time.time(): expire = e_time except Exception as _exp: pass except urllib2.HTTPError as httperror: msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers) data = {"error": "Unable to contact %s" % contact, "reason": msg} try: err = "%s %s" % (contact, extract_http_error(httperror.read())) data.update({"error": err}) msg += "\n" + err except Exception as exp: data.update({"httperror": None}) msg += "\n" + str(exp) print msg data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers) print msg + "\n" + str(exp) data = {"error": "Unable to contact %s" % contact, "reason": msg} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None format = '' if args: recordid = args[0] spec = {'_id':recordid} fields = None query = dict(fields=fields, spec=spec) if len(args) == 2: format = args[1] elif kwargs and '_id' in kwargs: spec = {'_id': kwargs['_id']} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) nresults = self.nresults(query) time0 = time.time() url = self.cachesrv idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) show = getarg(kwargs, 'show', 'json') coll = getarg(kwargs, 'collection', 'merge') # params = {'query':json.dumps(query), 'idx':idx, 'limit':limit} # path = '/rest/request' params = {'query':json.dumps(query), 'idx':idx, 'limit':limit, 'collection':coll} path = '/rest/records' headers = {"Accept": "application/json"} try: data = urllib2_request('GET', url+path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {'status':'fail', 'reason':traceback.format_exc()} res = "" if result['status'] == 'success': if recordid: # we got id for row in result['data']: if show == 'json': jsoncode = {'jsoncode': json2html(row, "")} res += self.templatepage('das_json', **jsoncode) elif show == 'code': code = pformat(row, indent=1, width=100) res += self.templatepage('das_code', code=code) else: code = yaml.dump(row, width=100, indent=4, default_flow_style=False) res += self.templatepage('das_code', code=code) else: for row in result['data']: rid = row['_id'] del row['_id'] record = dict(id=rid, daskeys=', '.join(row)) res += self.templatepage('das_record', **record) else: res = result['status'] if 'reason' in res: return self.error(res['reason']) else: msg = 'Uknown error, kwargs=' % kwargs return self.error(msg) if recordid: if format: if format == 'xml': return self.wrap2dasxml(result['data']) elif format == 'json': return self.wrap2dasjson(result['data']) else: return self.error('Unsupported data format %s' % format) page = res else: url = '/das/records?' idict = dict(nrows=nresults, idx=idx, limit=limit, results=res, url=url) page = self.templatepage('das_pagination', **idict) form = self.form(uinput="") ctime = (time.time()-time0) page = self.page(form + page, ctime=ctime) return page except: return self.error(self.gen_error_msg(kwargs))
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None if args: recordid = args[0] spec = {'_id':ObjectId(recordid)} fields = None query = dict(fields=fields, spec=spec) elif kwargs and '_id' in kwargs: spec = {'_id': ObjectId(kwargs['_id'])} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) res = '' time0 = time.time() idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 50) coll = kwargs.get('collection', 'merge') view = kwargs.get('view', '') if view == 'json': res = [] inst = kwargs.get('instance', self.dbs_global) form = self.form(uinput="") check, content = self.generate_dasquery(query, inst) if check: return self.page(form + content, ctime=time.time()-time0) dasquery = content # returned content is valid DAS query nresults = self.dasmgr.rawcache.nresults(dasquery, coll) gen = self.dasmgr.rawcache.get_from_cache\ (dasquery, idx=idx, limit=limit, collection=coll) if recordid: # we got id for row in gen: if view == 'json': res.append(row) else: res += das_json(dasquery, row) else: for row in gen: rid = row['_id'] del row['_id'] res += self.templatepage('das_record', \ id=rid, collection=coll, daskeys=', '.join(row)) if recordid: page = res else: url = '/das/records?' if nresults: page = self.templatepage('das_pagination', \ nrows=nresults, idx=idx, limit=limit, url=url, \ cgi=cgi, str=str) else: page = 'No results found, nresults=%s' % nresults page += res ctime = (time.time()-time0) if view == 'json': return json.dumps(res) page = self.page(form + page, ctime=ctime) return page except Exception as exc: print_exc(exc) return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None format = '' if args: recordid = args[0] spec = {'_id': recordid} fields = None query = dict(fields=fields, spec=spec) if len(args) == 2: format = args[1] elif kwargs and '_id' in kwargs: spec = {'_id': kwargs['_id']} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) nresults = self.nresults(query) time0 = time.time() url = self.cachesrv idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) show = getarg(kwargs, 'show', 'json') coll = getarg(kwargs, 'collection', 'merge') # params = {'query':json.dumps(query), 'idx':idx, 'limit':limit} # path = '/rest/request' params = { 'query': json.dumps(query), 'idx': idx, 'limit': limit, 'collection': coll } path = '/rest/records' headers = {"Accept": "application/json"} try: data = urllib2_request('GET', url + path, params, headers=headers) result = json.loads(data) except: self.daslogger.error(traceback.format_exc()) result = {'status': 'fail', 'reason': traceback.format_exc()} res = "" if result['status'] == 'success': if recordid: # we got id for row in result['data']: if show == 'json': jsoncode = {'jsoncode': json2html(row, "")} res += self.templatepage('das_json', **jsoncode) elif show == 'code': code = pformat(row, indent=1, width=100) res += self.templatepage('das_code', code=code) else: code = yaml.dump(row, width=100, indent=4, default_flow_style=False) res += self.templatepage('das_code', code=code) else: for row in result['data']: rid = row['_id'] del row['_id'] record = dict(id=rid, daskeys=', '.join(row)) res += self.templatepage('das_record', **record) else: res = result['status'] if 'reason' in res: return self.error(res['reason']) else: msg = 'Uknown error, kwargs=' % kwargs return self.error(msg) if recordid: if format: if format == 'xml': return self.wrap2dasxml(result['data']) elif format == 'json': return self.wrap2dasjson(result['data']) else: return self.error('Unsupported data format %s' % format) page = res else: url = '/das/records?' idict = dict(nrows=nresults, idx=idx, limit=limit, results=res, url=url) page = self.templatepage('das_pagination', **idict) form = self.form(uinput="") ctime = (time.time() - time0) page = self.page(form + page, ctime=ctime) return page except: return self.error(self.gen_error_msg(kwargs))
def set_misses(self, dasquery, api, genrows): """ Check and adjust DAS records wrt input query. If some of the DAS keys are missing, add it with its value to the DAS record. """ # look-up primary key prim_key = self.dasmapping.primary_key(self.name, api) # Scan all docs and store those whose size above MongoDB limit into # GridFS map_key = self.dasmapping.primary_mapkey(self.name, api) genrows = parse2gridfs(self.gfs, map_key, genrows, self.logger) spec = dasquery.mongo_query['spec'] row = next(genrows) ddict = DotDict(row) keys2adjust = [] for key in spec.keys(): val = ddict.get(key) if spec[key] != val and key not in keys2adjust: keys2adjust.append(key) msg = "adjust keys %s" % keys2adjust self.logger.debug(msg) count = 0 if keys2adjust: # adjust of the rows for row in yield_rows(row, genrows): ddict = DotDict(row) pval = ddict.get(map_key) if isinstance(pval, dict) and 'error' in pval: ddict[map_key] = '' ddict.update({prim_key: pval}) for key in keys2adjust: value = spec[key] existing_value = ddict.get(key) # the way to deal with proximity/patern/condition results if (isinstance(value, str) or isinstance(value, unicode))\ and value.find('*') != -1: # we got pattern if existing_value: value = existing_value elif isinstance(value, dict) or \ isinstance(value, list): # we got condition if existing_value: value = existing_value elif isinstance(value, dict) and \ '$in' in value: # we got a range {'$in': []} value = value['$in'] elif isinstance(value, dict) and \ '$lte' in value and '$gte' in value: # we got a between range value = [value['$gte'], value['$lte']] else: value = json.dumps(value) elif existing_value and value != existing_value: # we got proximity results if 'proximity' in ddict: proximity = DotDict({key:existing_value}) ddict['proximity'].update(proximity) else: proximity = DotDict({}) proximity[key] = existing_value ddict['proximity'] = proximity else: if existing_value: value = existing_value ddict[key] = value yield ddict count += 1 else: yield row for row in genrows: yield row count += 1 msg = "yield %s rows" % count self.logger.debug(msg)
def getdata_urllib(url, params, headers=None, expire=3600, post=None, error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None, tstamp=None): """ Invoke URL call and retrieve data from data-service based on provided URL and set of parameters. Use post=True to invoke POST request. """ contact = 'data-service.' if system: contact = system + ' ' + contact timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True)) das_timer(timer_key, verbose) encoded_data = urllib.urlencode(params, doseq=doseq) if not post: url = url + '?' + encoded_data if not headers: headers = {} if tstamp and 'If-Modified-Since' not in headers.keys(): headers['If-Modified-Since'] = http_timestamp(tstamp) if verbose: print('+++ getdata, url=%s, headers=%s' % (url, headers)) req = urllib2.Request(url) for key, val in headers.items(): req.add_header(key, val) if verbose > 1: handler = urllib2.HTTPHandler(debuglevel=1) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) if ckey and cert: handler = HTTPSClientAuthHandler(ckey, cert, verbose) opener = urllib2.build_opener(handler) urllib2.install_opener(opener) try: time0 = time.time() if post: data = urllib2.urlopen(req, encoded_data) else: data = urllib2.urlopen(req) data_srv_time = time.time()-time0 info = data.info() code = data.getcode() if verbose > 1: print("+++ response code:", code) print("+++ response info\n", info) try: # get HTTP header and look for Expires e_time = expire_timestamp(\ info.__dict__['dict']['expires']) if e_time < expire_timestamp(data_srv_time): expire = max(e_time, expire_timestamp(expire)) elif e_time > time.time(): expire = e_time except Exception as _exp: pass except urllib2.HTTPError as httperror: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) data = {'error': 'Received HTTP error from %s data-service' % contact, 'reason': msg} try: err = '%s %s' % (contact, extract_http_error(httperror.read())) data.update({'error':err}) msg += '\n' + err except Exception as exp: data.update({'httperror': None}) msg += '\n' + str(exp) print(msg) data = json.dumps(data) expire = expire_timestamp(error_expire) except Exception as exp: msg = 'HTTPError, url=%s, args=%s, headers=%s' \ % (url, params, headers) print(msg + '\n' + str(exp)) data = {'error': 'Received generic error from %s data-service' % contact, 'reason': msg} data = json.dumps(data) expire = expire_timestamp(error_expire) das_timer(timer_key, verbose) return data, expire