def test_yajl(self):
"""Test yajl wrapper"""
myjson.MODULE = 'yajl'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
data = {'a':1, 'b':2}
kwds = {'sort_keys':True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b':2, 'a':1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = 123
result1 = myjson.JSONEncoder(**kwds).encode(data)
kwds = {'sort_keys':True}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}}
result1 = myjson.JSONEncoder().encode(data)
result2 = myjson.JSONDecoder().decode(result1)
self.assertEqual(data, result2)
data = {'a':123, 'b':[1, 2, 3], 'c':{'d':[1, '2', 3]}}
result1 = myjson.dumps(data)
result2 = myjson.loads(result1)
self.assertEqual(data, result2)
myjson.MODULE = self.module
def encode_mongo_query(query):
"""
Encode mongo query into storage format. MongoDB does not allow storage of
dict with keys containing "." or MongoDB operators, e.g. $lt. So we
convert input mongo query spec into list of dicts whose "key"/"value"
are mongo query spec key/values. For example
.. doctest::
spec:{"block.name":"aaa"}
converted into
spec:[{"key":"block.name", "value":'"aaa"'}]
Conversion is done using JSON dumps method.
"""
if not query:
msg = 'Cannot decode, input query=%s' % query
raise Exception(msg)
return_query = dict(query)
speclist = []
for key, val in return_query.pop('spec').items():
if str(type(val)) == "<type '_sre.SRE_Pattern'>":
val = json.dumps(val.pattern)
speclist.append({"key":key, "value":val, "pattern":1})
else:
val = json.dumps(val)
speclist.append({"key":key, "value":val})
return_query['spec'] = speclist
return return_query
def keys(self, **kwds):
"""
Show DAS keys and their attibutes
"""
adict = {}
for row in self.dasmgr.keylearning.attributes():
try:
qpat = row.get('query_pat', [])
key, attr = row['member'].split('.', 1)
except:
continue
if key in adict:
vdict = adict[key]
if attr in vdict:
vdict[attr] += qpat
else:
vdict[attr] = qpat
adict[key] = vdict
else:
adict[key] = {attr: qpat}
view = kwds.get('view', '')
if view == 'json':
return json.dumps(adict)
page = self.templatepage('das_keys_attrs', attrs=adict)
return self.page(page, response_div=False)
def das2api(self, system, api, rec_key, value=None):
"""
Translates DAS record key into data-service API input parameter,
e.g. run.number => run_number
"""
entry = self.dasmapscache.get((system, api), None)
names = []
if not entry:
return [rec_key]
for row in entry.get('das_map', []):
if 'api_arg' in row:
api_param = row['api_arg']
pat = row.get('pattern', None)
if row['rec_key'] != rec_key:
continue
if value and pat:
if isinstance(value, dict):
if pat.match(json.dumps(value.values()[0])):
if api_param not in names:
names.append(api_param)
if pat.match(str(value)):
if api_param not in names:
names.append(api_param)
else:
if api_param not in names:
names.append(api_param)
else:
names.append(row['rec_key'])
return names
def form(self, uinput="", instance=None, view="list", cards=False):
"""
provide input DAS search form
"""
# TODO: rename into search_form()? (template is also called like this
if "'" in uinput: # e.g. file.creation_date>'20120101 12:01:01'
uinput = uinput.replace("'", '"')
if not instance:
instance = self.dbs_global
cards = self.templatepage(
"das_cards", base=self.base, show=cards, width=900, height=220, cards=help_cards(self.base)
)
daskeys = self.templatepage("das_keys", daskeys=self.daskeyslist)
page = self.templatepage(
"das_searchform",
input=uinput,
init_dbses=list(self.dbs_instances),
daskeys=daskeys,
base=self.base,
instance=instance,
view=view,
cards=cards,
autocompl_host=json.dumps(self._get_autocompl_host()),
)
return page
def worker_v3(url, query):
"""
Query RunRegistry service, see documentation at
https://twiki.cern.ch/twiki/bin/viewauth/CMS/DqmRrApi
url=http://runregistry.web.cern.ch/runregistry/
"""
workspace = 'GLOBAL'
table = 'runsummary'
template = 'json'
columns = ['number', 'startTime', 'stopTime', 'triggers',
'runClassName', 'runStopReason', 'bfield', 'gtKey',
'l1Menu', 'hltKeyDescription', 'lhcFill', 'lhcEnergy',
'runCreated', 'modified', 'lsCount', 'lsRanges']
sdata = json.dumps({'filter':query})
path = 'api/GLOBAL/%s/%s/%s/none/data' \
% (table, template, urllib.quote(','.join(columns)))
callurl = os.path.join(url, path)
result = urllib.urlopen(callurl, sdata)
record = json.load(result)
result.close()
notations = {'lsRanges':'lumi_section_ranges',
'number':'run_number', 'runCreated':'create_time',
'stopTime': 'end_time', 'startTime': 'start_time',
'lsCount': 'lumi_sections', 'runStopReason': 'stop_reason',
'hltKeyDescription': 'hltkey', 'gtKey': 'gtkey',
'lhcEnergy': 'beam_e', 'l1Menu': 'l1key',
'modified': 'modify_time', 'runClassName': 'group_name'}
for rec in record:
for key, val in rec.items():
if notations.has_key(key):
rec[notations[key]] = val
del rec[key]
yield dict(run=rec)
def storage_query(self):
"""
Read only storage query, generated on demand.
"""
if not self._storage_query:
self._storage_query = deepcopy(self.mongo_query)
speclist = []
for key, val in self._storage_query.pop('spec').items():
if str(type(val)) == "<type '_sre.SRE_Pattern'>":
val = json.dumps(val.pattern)
speclist.append({"key":key, "value":val, "pattern":1})
elif isinstance(val, ObjectId):
speclist.append({"key":key, "value":str(val)})
else:
val = json.dumps(val)
speclist.append({"key":key, "value":val})
self._storage_query['spec'] = speclist
return self._storage_query
def test_cjson(self):
"""Test cjson wrapper"""
myjson.MODULE = 'cjson'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
myjson.MODULE = self.module
def dump(ilist, idx=0):
"""
Print items in provided generator
"""
if isinstance(ilist, GeneratorType):
reslist = ilist
elif not isinstance(ilist, list):
reslist = [ilist]
else:
reslist = ilist
if not reslist:
print "dump, no results found"
return
idx = 0
for row in reslist:
print "id : %s" % idx
if isinstance(row, dict):
print json.dumps(row)
else:
print row
print
idx += 1
def check_map_record(rec):
"Check hash of given map record"
# remove _id MongoDB Object
if "_id" in rec:
del rec["_id"]
if "hash" in rec:
md5 = rec.pop("hash")
rec_md5 = md5hash(rec)
if rec_md5 != md5:
err = "Invalid hash record:\n%s\n" % json.dumps(rec)
err += "\nrecord hash : %s" % md5
err += "\nobtained hash: %s\n" % md5hash(rec)
raise Exception(err)
def _test_yajl(self):
"""Test yajl wrapper"""
import yajl
myjson.MODULE = 'yajl'
myjson.yajl = yajl # so not to depend on import sequence
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
data = {'a': 1, 'b': 2}
kwds = {'sort_keys': True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b': 2, 'a': 1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = 123
result1 = myjson.JSONEncoder(**kwds).encode(data)
kwds = {'sort_keys': True}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}}
result1 = myjson.JSONEncoder().encode(data)
result2 = myjson.JSONDecoder().decode(result1)
self.assertEqual(data, result2)
data = {'a': 123, 'b': [1, 2, 3], 'c': {'d': [1, '2', 3]}}
result1 = myjson.dumps(data)
result2 = myjson.loads(result1)
self.assertEqual(data, result2)
myjson.MODULE = self.module
def check_map_record(record):
"Check hash of given map record"
rec = dict(record)
# remove _id MongoDB Object
for key in ['_id']+TRANSIENT_FIELDS:
if key in rec:
del rec[key]
if 'hash' in rec:
md5 = rec.pop('hash')
rec_md5 = md5hash(rec)
if rec_md5 != md5:
err = 'Invalid hash record:\n%s\n' % json.dumps(rec)
err += '\nrecord hash : %s' % md5
err += '\nobtained hash: %s\n' % md5hash(rec)
raise Exception(err)
def _test_cjson(self):
""" Test cjson wrapper """
import cjson
myjson.MODULE = 'cjson'
myjson.cjson = cjson
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
myjson.MODULE = self.module
def getdata_pycurl(
url,
params,
headers=None,
expire=3600,
post=None,
error_expire=300,
verbose=0,
ckey=None,
cert=None,
doseq=True,
system=None,
):
"Fetch data via pycurl library"
contact = "data-service."
if system:
contact = system + " " + contact
timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
handler = REQUEST_HANDLER
try:
data, expire = handler.getdata(url, params, headers, expire, post, error_expire, verbose, ckey, cert, doseq)
except urllib2.HTTPError as httperror:
msg = "urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s" % (
system,
url,
json.dumps(params),
json.dumps(headers),
)
data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()}
try:
reason = extract_http_error(httperror.read())
data.update({"reason": reason, "request": msg})
msg += "\n" + err
except Exception as exp:
data.update({"httperror": None})
msg += "\n" + str(exp)
print dastimestamp("getdata_pycurl"), msg
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = "HTTPError, system=%s, url=%s, args=%s, headers=%s" % (
system,
url,
json.dumps(params),
json.dumps(headers),
)
print dastimestamp("getdata_pycurl"), msg + "\n" + str(exp)
data = {"error": "Unable to contact %s" % contact, "reason": msg, "ts": time.time()}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
def gridfs(self, **kwargs):
"""
Retieve records from GridFS
"""
time0 = time.time()
if 'fid' not in kwargs:
code = web_code('No file id')
raise HTTPError(500, 'DAS error, code=%s' % code)
fid = kwargs.get('fid')
data = {'status':'requested', 'fid':fid}
try:
fds = self.gfs.get(ObjectId(fid))
return fds.read()
except Exception as exc:
print_exc(exc)
code = web_code('Exception')
raise HTTPError(500, 'DAS error, code=%s' % code)
data['ctime'] = time.time() - time0
return json.dumps(data)
def gridfs(self, **kwargs):
"""
Retieve records from GridFS
"""
time0 = time.time()
if "fid" not in kwargs:
code = web_code("No file id")
raise HTTPError(500, "DAS error, code=%s" % code)
fid = kwargs.get("fid")
data = {"status": "requested", "fid": fid}
try:
fds = self.gfs.get(ObjectId(fid))
return fds.read()
except Exception as exc:
print_exc(exc)
code = web_code("Exception")
raise HTTPError(500, "DAS error, code=%s" % code)
data["ctime"] = time.time() - time0
return json.dumps(data)
def test_json(self):
"""Test json wrapper"""
myjson.MODULE = 'json'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(self.data))
self.assertEqual(expect, result)
data = {'a':1, 'b':2}
kwds = {'sort_keys':True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b':2, 'a':1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
myjson.MODULE = self.module
def _test_json(self):
""" Test json wrapper """
myjson.MODULE = 'json'
expect = self.data
result = myjson.loads(myjson.dumps(self.data))
self.assertEqual(expect, result)
expect = self.data
result = myjson.JSONDecoder().decode(myjson.JSONEncoder().encode(
self.data))
self.assertEqual(expect, result)
data = {'a': 1, 'b': 2}
kwds = {'sort_keys': True}
result1 = myjson.JSONEncoder(**kwds).encode(data)
data = {'b': 2, 'a': 1}
result2 = myjson.JSONEncoder(**kwds).encode(data)
self.assertEqual(result1, result2)
myjson.MODULE = self.module
def urllib2_request(request, url, params, headers=None, debug=0):
"""request method using GET request from urllib2 library"""
if not headers:
headers = {}
if request == "GET" or request == "DELETE":
encoded_data = urllib.urlencode(params, doseq=True)
url += "?%s" % encoded_data
req = UrlRequest(request, url=url, headers=headers)
else:
encoded_data = json.dumps(params)
req = UrlRequest(request, url=url, data=encoded_data, headers=headers)
if debug:
hdlr = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(hdlr)
else:
opener = urllib2.build_opener()
fdesc = opener.open(req)
result = fdesc.read()
fdesc.close()
return result
def blocks4tier_date(dbs, tier, min_cdate, max_cdate, verbose=0):
"Get list of blocks for given parameters"
headers = {"Accept": "text/json;application/json"}
url = dbs + "/blocks"
params = {"data_tier_name": tier, "min_cdate": min_cdate, "max_cdate": max_cdate}
urls = ["%s?%s" % (url, urllib.urlencode(params))]
if verbose > 1:
print "\nblocks4tier_date"
print urls
res = process(urlfetch_getdata(urls, CKEY, CERT, headers))
err = "Unable to get blocks for tier=%s, mindate=%s, maxdate=%s" % (tier, min_cdate, max_cdate)
for blist in res:
if "error" in blist:
yield blist
continue
if isinstance(blist, dict):
if "block_name" not in blist:
msg = err + ", reason=%s" % json.dumps(blist)
raise Exception(msg)
for row in blist:
yield row["block_name"]
def form(self, uinput='', instance=None, view='list', cards=False):
"""
provide input DAS search form
"""
# TODO: rename into search_form()? (template is also called like this
if "'" in uinput: # e.g. file.creation_date>'20120101 12:01:01'
uinput = uinput.replace("'", '"')
if not instance:
instance = self.dbs_global
hcards = help_cards(self.base)
width = 900
height = 220
cards = self.templatepage('das_cards', base=self.base, show=cards, \
width=width, height=height, max_width=len(hcards)*width, \
cards=hcards, enumerate=enumerate)
daskeys = self.templatepage('das_keys', daskeys=self.daskeyslist)
page = self.templatepage('das_searchform', input=uinput, \
init_dbses=list(self.dbs_instances), daskeys=daskeys, \
base=self.base, instance=instance, view=view, cards=cards,
autocompl_host=json.dumps(self._get_autocompl_host())
)
return page
def wrapper (self, *args, **kwds):
"""Decorator wrapper"""
cherrypy.response.headers['Content-Type'] = "application/json"
func._cp_config = {'response.stream': True}
head, data = func (self, *args, **kwds)
yield json.dumps(head)[:-1] # do not yield }
yield ', "data": ['
if isinstance(data, dict):
for chunk in JSONEncoder().iterencode(data):
yield chunk
elif isinstance(data, list) or isinstance(data, types.GeneratorType):
sep = ''
for rec in data:
if sep:
yield sep
for chunk in JSONEncoder().iterencode(rec):
yield chunk
if not sep:
sep = ', '
else:
msg = 'jsonstreamer, improper data type %s' % type(data)
raise Exception(msg)
yield ']}'
def add(self, pid, kwds):
"""Add new pid/kwds"""
self.clean()
if not kwds:
return
tstamp = time.strftime("%Y%m%d %H:%M:%S", time.localtime())
doc = dict(_id=pid, kwds=json.dumps(kwds),
ts=time.time(), timestamp=tstamp)
attempts = 0
while True:
try:
self.col.insert(doc, safe=True)
break
except DuplicateKeyError as err:
break
except Exception as err:
print_exc(err)
time.sleep(0.01)
attempts += 1
if attempts > 2:
msg = '%s unable to add pid=%s' % (self.col, pid)
print dastimestamp('DAS ERROR '), msg
break
def blocks4tier_date(dbs, tier, min_cdate, max_cdate, verbose=0):
"Get list of blocks for given parameters"
headers = {'Accept':'text/json;application/json'}
url = dbs + "/blocks"
params = {'data_tier_name':tier,
'min_cdate':min_cdate,
'max_cdate':max_cdate}
urls = ['%s?%s' % (url, urllib.urlencode(params))]
if verbose > 1:
print("\nblocks4tier_date")
print(urls)
res = process(urlfetch_getdata(urls, CKEY, CERT, headers))
err = 'Unable to get blocks for tier=%s, mindate=%s, maxdate=%s' \
% (tier, min_cdate, max_cdate)
for blist in res:
if 'error' in blist:
yield blist
continue
if isinstance(blist, dict):
if 'block_name' not in blist:
msg = err + ', reason=%s' % json.dumps(blist)
raise Exception(msg)
for row in blist:
yield row['block_name']
def getdata_pycurl(url, params, headers=None, expire=3600, post=None,
error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None):
"Fetch data via pycurl library"
contact = 'data-service.'
if system:
contact = system + ' ' + contact
if isinstance(params, dict):
timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
else:
timer_key = '%s?%s' % (url, params)
das_timer(timer_key, verbose)
handler = REQUEST_HANDLER
try:
data, expire = handler.getdata(url, params, headers, expire, post, \
error_expire, verbose, ckey, cert, doseq)
except urllib2.HTTPError as httperror:
msg = 'urllib2.HTTPError, system=%s, url=%s, args=%s, headers=%s' \
% (system, url, json.dumps(params), json.dumps(headers))
data = {'error': 'Received HTTP error from %s data-service' % contact,
'reason': msg, 'ts':time.time()}
try:
reason = extract_http_error(httperror.read())
data.update({'reason': reason, 'request': msg})
# TODO: err variable did not exit in this function!
msg += '\n' + reason
except Exception as exp:
data.update({'httperror': None})
msg += '\n' + str(exp)
print(dastimestamp('getdata_pycurl'), msg)
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = 'HTTPError, system=%s, url=%s, args=%s, headers=%s' \
% (system, url, json.dumps(params), json.dumps(headers))
print(dastimestamp('getdata_pycurl'), msg + '\n' + str(exp))
data = {'error': 'Received generic error from %s data-service' % contact,
'reason': msg, 'ts':time.time()}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
if args:
recordid = args[0]
spec = {"_id": ObjectId(recordid)}
fields = None
query = dict(fields=fields, spec=spec)
elif kwargs and "_id" in kwargs:
spec = {"_id": ObjectId(kwargs["_id"])}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
res = ""
time0 = time.time()
idx = getarg(kwargs, "idx", 0)
limit = getarg(kwargs, "limit", 10)
coll = kwargs.get("collection", "merge")
view = kwargs.get("view", "")
if view == "json":
res = []
inst = kwargs.get("instance", self.dbs_global)
form = self.form(uinput="")
check, content = self.generate_dasquery(query, inst)
if check:
return self.page(form + content, ctime=time.time() - time0)
dasquery = content # returned content is valid DAS query
nresults = self.dasmgr.rawcache.nresults(dasquery, coll)
gen = self.dasmgr.rawcache.get_from_cache(dasquery, idx=idx, limit=limit, collection=coll)
if recordid: # we got id
for row in gen:
if view == "json":
res.append(row)
else:
res += das_json(dasquery, row)
else:
for row in gen:
rid = row["_id"]
del row["_id"]
res += self.templatepage("das_record", id=rid, collection=coll, daskeys=", ".join(row))
if recordid:
page = res
else:
url = "/das/records?"
if nresults:
page = self.templatepage("das_pagination", nrows=nresults, idx=idx, limit=limit, url=url)
else:
page = "No results found, nresults=%s" % nresults
page += res
ctime = time.time() - time0
if view == "json":
return json.dumps(res)
page = self.page(form + page, ctime=ctime)
return page
except Exception as exc:
print_exc(exc)
return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ""
if args:
recordid = args[0]
spec = {"_id": recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and kwargs.has_key("_id"):
spec = {"_id": kwargs["_id"]}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, "idx", 0)
limit = getarg(kwargs, "limit", 10)
show = getarg(kwargs, "show", "json")
coll = getarg(kwargs, "collection", "merge")
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {"query": json.dumps(query), "idx": idx, "limit": limit, "collection": coll}
path = "/rest/records"
headers = {"Accept": "application/json"}
try:
data = urllib2_request("GET", url + path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {"status": "fail", "reason": traceback.format_exc()}
res = ""
if result["status"] == "success":
if recordid: # we got id
for row in result["data"]:
if show == "json":
jsoncode = {"jsoncode": json2html(row, "")}
res += self.templatepage("das_json", **jsoncode)
elif show == "code":
code = pformat(row, indent=1, width=100)
res += self.templatepage("das_code", code=code)
else:
code = yaml.dump(row, width=100, indent=4, default_flow_style=False)
res += self.templatepage("das_code", code=code)
else:
for row in result["data"]:
rid = row["_id"]
del row["_id"]
record = dict(id=rid, daskeys=", ".join(row))
res += self.templatepage("das_record", **record)
else:
res = result["status"]
if res.has_key("reason"):
return self.error(res["reason"])
else:
msg = "Uknown error, kwargs=" % kwargs
return self.error(msg)
if recordid:
if format:
if format == "xml":
return self.wrap2dasxml(result["data"])
elif format == "json":
return self.wrap2dasjson(result["data"])
else:
return self.error("Unsupported data format %s" % format)
page = res
else:
url = "/das/records?"
idict = dict(nrows=nresults, idx=idx, limit=limit, results=res, url=url)
page = self.templatepage("das_pagination", **idict)
form = self.form(uinput="")
ctime = time.time() - time0
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def getdata_urllib(
url,
params,
headers=None,
expire=3600,
post=None,
error_expire=300,
verbose=0,
ckey=None,
cert=None,
doseq=True,
system=None,
):
"""
Invoke URL call and retrieve data from data-service based
on provided URL and set of parameters. Use post=True to
invoke POST request.
"""
contact = "data-service."
if system:
contact = system + " " + contact
timer_key = "%s?%s" % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + "?" + encoded_data
if not headers:
headers = {}
if verbose:
print "+++ getdata, url=%s, headers=%s" % (url, headers)
req = urllib2.Request(url)
for key, val in headers.iteritems():
req.add_header(key, val)
if verbose > 1:
handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if ckey and cert:
handler = HTTPSClientAuthHandler(ckey, cert, verbose)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
try:
time0 = time.time()
if post:
data = urllib2.urlopen(req, encoded_data)
else:
data = urllib2.urlopen(req)
data_srv_time = time.time() - time0
try: # get HTTP header and look for Expires
e_time = expire_timestamp(data.info().__dict__["dict"]["expires"])
if e_time < expire_timestamp(data_srv_time):
expire = max(e_time, expire_timestamp(expire))
elif e_time > time.time():
expire = e_time
except Exception as _exp:
pass
except urllib2.HTTPError as httperror:
msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
data = {"error": "Unable to contact %s" % contact, "reason": msg}
try:
err = "%s %s" % (contact, extract_http_error(httperror.read()))
data.update({"error": err})
msg += "\n" + err
except Exception as exp:
data.update({"httperror": None})
msg += "\n" + str(exp)
print msg
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = "HTTPError, url=%s, args=%s, headers=%s" % (url, params, headers)
print msg + "\n" + str(exp)
data = {"error": "Unable to contact %s" % contact, "reason": msg}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ''
if args:
recordid = args[0]
spec = {'_id':recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and '_id' in kwargs:
spec = {'_id': kwargs['_id']}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
coll = getarg(kwargs, 'collection', 'merge')
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {'query':json.dumps(query), 'idx':idx, 'limit':limit,
'collection':coll}
path = '/rest/records'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url+path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status':'fail', 'reason':traceback.format_exc()}
res = ""
if result['status'] == 'success':
if recordid: # we got id
for row in result['data']:
if show == 'json':
jsoncode = {'jsoncode': json2html(row, "")}
res += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(row, indent=1, width=100)
res += self.templatepage('das_code', code=code)
else:
code = yaml.dump(row, width=100, indent=4,
default_flow_style=False)
res += self.templatepage('das_code', code=code)
else:
for row in result['data']:
rid = row['_id']
del row['_id']
record = dict(id=rid, daskeys=', '.join(row))
res += self.templatepage('das_record', **record)
else:
res = result['status']
if 'reason' in res:
return self.error(res['reason'])
else:
msg = 'Uknown error, kwargs=' % kwargs
return self.error(msg)
if recordid:
if format:
if format == 'xml':
return self.wrap2dasxml(result['data'])
elif format == 'json':
return self.wrap2dasjson(result['data'])
else:
return self.error('Unsupported data format %s' % format)
page = res
else:
url = '/das/records?'
idict = dict(nrows=nresults, idx=idx,
limit=limit, results=res, url=url)
page = self.templatepage('das_pagination', **idict)
form = self.form(uinput="")
ctime = (time.time()-time0)
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
if args:
recordid = args[0]
spec = {'_id':ObjectId(recordid)}
fields = None
query = dict(fields=fields, spec=spec)
elif kwargs and '_id' in kwargs:
spec = {'_id': ObjectId(kwargs['_id'])}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
res = ''
time0 = time.time()
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 50)
coll = kwargs.get('collection', 'merge')
view = kwargs.get('view', '')
if view == 'json':
res = []
inst = kwargs.get('instance', self.dbs_global)
form = self.form(uinput="")
check, content = self.generate_dasquery(query, inst)
if check:
return self.page(form + content, ctime=time.time()-time0)
dasquery = content # returned content is valid DAS query
nresults = self.dasmgr.rawcache.nresults(dasquery, coll)
gen = self.dasmgr.rawcache.get_from_cache\
(dasquery, idx=idx, limit=limit, collection=coll)
if recordid: # we got id
for row in gen:
if view == 'json':
res.append(row)
else:
res += das_json(dasquery, row)
else:
for row in gen:
rid = row['_id']
del row['_id']
res += self.templatepage('das_record', \
id=rid, collection=coll, daskeys=', '.join(row))
if recordid:
page = res
else:
url = '/das/records?'
if nresults:
page = self.templatepage('das_pagination', \
nrows=nresults, idx=idx, limit=limit, url=url, \
cgi=cgi, str=str)
else:
page = 'No results found, nresults=%s' % nresults
page += res
ctime = (time.time()-time0)
if view == 'json':
return json.dumps(res)
page = self.page(form + page, ctime=ctime)
return page
except Exception as exc:
print_exc(exc)
return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ''
if args:
recordid = args[0]
spec = {'_id': recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and '_id' in kwargs:
spec = {'_id': kwargs['_id']}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
coll = getarg(kwargs, 'collection', 'merge')
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {
'query': json.dumps(query),
'idx': idx,
'limit': limit,
'collection': coll
}
path = '/rest/records'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET',
url + path,
params,
headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status': 'fail', 'reason': traceback.format_exc()}
res = ""
if result['status'] == 'success':
if recordid: # we got id
for row in result['data']:
if show == 'json':
jsoncode = {'jsoncode': json2html(row, "")}
res += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(row, indent=1, width=100)
res += self.templatepage('das_code', code=code)
else:
code = yaml.dump(row,
width=100,
indent=4,
default_flow_style=False)
res += self.templatepage('das_code', code=code)
else:
for row in result['data']:
rid = row['_id']
del row['_id']
record = dict(id=rid, daskeys=', '.join(row))
res += self.templatepage('das_record', **record)
else:
res = result['status']
if 'reason' in res:
return self.error(res['reason'])
else:
msg = 'Uknown error, kwargs=' % kwargs
return self.error(msg)
if recordid:
if format:
if format == 'xml':
return self.wrap2dasxml(result['data'])
elif format == 'json':
return self.wrap2dasjson(result['data'])
else:
return self.error('Unsupported data format %s' %
format)
page = res
else:
url = '/das/records?'
idict = dict(nrows=nresults,
idx=idx,
limit=limit,
results=res,
url=url)
page = self.templatepage('das_pagination', **idict)
form = self.form(uinput="")
ctime = (time.time() - time0)
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def set_misses(self, dasquery, api, genrows):
"""
Check and adjust DAS records wrt input query. If some of the DAS
keys are missing, add it with its value to the DAS record.
"""
# look-up primary key
prim_key = self.dasmapping.primary_key(self.name, api)
# Scan all docs and store those whose size above MongoDB limit into
# GridFS
map_key = self.dasmapping.primary_mapkey(self.name, api)
genrows = parse2gridfs(self.gfs, map_key, genrows, self.logger)
spec = dasquery.mongo_query['spec']
row = next(genrows)
ddict = DotDict(row)
keys2adjust = []
for key in spec.keys():
val = ddict.get(key)
if spec[key] != val and key not in keys2adjust:
keys2adjust.append(key)
msg = "adjust keys %s" % keys2adjust
self.logger.debug(msg)
count = 0
if keys2adjust:
# adjust of the rows
for row in yield_rows(row, genrows):
ddict = DotDict(row)
pval = ddict.get(map_key)
if isinstance(pval, dict) and 'error' in pval:
ddict[map_key] = ''
ddict.update({prim_key: pval})
for key in keys2adjust:
value = spec[key]
existing_value = ddict.get(key)
# the way to deal with proximity/patern/condition results
if (isinstance(value, str) or isinstance(value, unicode))\
and value.find('*') != -1: # we got pattern
if existing_value:
value = existing_value
elif isinstance(value, dict) or \
isinstance(value, list): # we got condition
if existing_value:
value = existing_value
elif isinstance(value, dict) and \
'$in' in value: # we got a range {'$in': []}
value = value['$in']
elif isinstance(value, dict) and \
'$lte' in value and '$gte' in value:
# we got a between range
value = [value['$gte'], value['$lte']]
else:
value = json.dumps(value)
elif existing_value and value != existing_value:
# we got proximity results
if 'proximity' in ddict:
proximity = DotDict({key:existing_value})
ddict['proximity'].update(proximity)
else:
proximity = DotDict({})
proximity[key] = existing_value
ddict['proximity'] = proximity
else:
if existing_value:
value = existing_value
ddict[key] = value
yield ddict
count += 1
else:
yield row
for row in genrows:
yield row
count += 1
msg = "yield %s rows" % count
self.logger.debug(msg)
def getdata_urllib(url, params, headers=None, expire=3600, post=None,
error_expire=300, verbose=0, ckey=None, cert=None, doseq=True, system=None,
tstamp=None):
"""
Invoke URL call and retrieve data from data-service based
on provided URL and set of parameters. Use post=True to
invoke POST request.
"""
contact = 'data-service.'
if system:
contact = system + ' ' + contact
timer_key = '%s?%s' % (url, urllib.urlencode(params, doseq=True))
das_timer(timer_key, verbose)
encoded_data = urllib.urlencode(params, doseq=doseq)
if not post:
url = url + '?' + encoded_data
if not headers:
headers = {}
if tstamp and 'If-Modified-Since' not in headers.keys():
headers['If-Modified-Since'] = http_timestamp(tstamp)
if verbose:
print('+++ getdata, url=%s, headers=%s' % (url, headers))
req = urllib2.Request(url)
for key, val in headers.items():
req.add_header(key, val)
if verbose > 1:
handler = urllib2.HTTPHandler(debuglevel=1)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
if ckey and cert:
handler = HTTPSClientAuthHandler(ckey, cert, verbose)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
try:
time0 = time.time()
if post:
data = urllib2.urlopen(req, encoded_data)
else:
data = urllib2.urlopen(req)
data_srv_time = time.time()-time0
info = data.info()
code = data.getcode()
if verbose > 1:
print("+++ response code:", code)
print("+++ response info\n", info)
try: # get HTTP header and look for Expires
e_time = expire_timestamp(\
info.__dict__['dict']['expires'])
if e_time < expire_timestamp(data_srv_time):
expire = max(e_time, expire_timestamp(expire))
elif e_time > time.time():
expire = e_time
except Exception as _exp:
pass
except urllib2.HTTPError as httperror:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
data = {'error': 'Received HTTP error from %s data-service' % contact,
'reason': msg}
try:
err = '%s %s' % (contact, extract_http_error(httperror.read()))
data.update({'error':err})
msg += '\n' + err
except Exception as exp:
data.update({'httperror': None})
msg += '\n' + str(exp)
print(msg)
data = json.dumps(data)
expire = expire_timestamp(error_expire)
except Exception as exp:
msg = 'HTTPError, url=%s, args=%s, headers=%s' \
% (url, params, headers)
print(msg + '\n' + str(exp))
data = {'error': 'Received generic error from %s data-service' % contact,
'reason': msg}
data = json.dumps(data)
expire = expire_timestamp(error_expire)
das_timer(timer_key, verbose)
return data, expire
