class DASSearch(DASWebManager):
"""
DAS web interface.
"""
def __init__(self, config={}):
DASWebManager.__init__(self, config)
try:
# try what is supplied from WebTools framework
cdict = self.config.dictionary_()
self.cachesrv = cdict.get('cache_server_url',
'http://localhost:8211')
self.base = '/dascontrollers'
except:
# stand-alone version
self.cachesrv = config.get('cache_server_url',
'http://localhost:8211')
self.base = '/das'
self.dasmgr = DASCore()
self.daskeys = self.dasmgr.das_keys()
self.daskeys.sort()
self.dasmapping = self.dasmgr.mapping
self.daslogger = self.dasmgr.logger
self.pageviews = ['xml', 'list', 'json', 'yuijson']
msg = "DASSearch::init is started with base=%s" % self.base
self.daslogger.debug(msg)
print(msg)
def top(self):
"""
Define masthead for all DAS web pages
"""
return self.templatepage('das_top', base=self.base)
def bottom(self, response_div=True):
"""
Define footer for all DAS web pages
"""
return self.templatepage('das_bottom', div=response_div)
def page(self, content, ctime=None, response_div=True):
"""
Define footer for all DAS web pages
"""
page = self.top()
page += content
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
services = self.dasmgr.keys()
srv = ""
for key in services.keys():
srv += "%s, " % key
srv = srv[:-2] # remove last comma
page += self.templatepage('das_bottom', ctime=ctime, services=srv,
timestamp=timestamp, div=response_div)
return page
@expose
def faq(self, *args, **kwargs):
"""
represent DAS FAQ.
"""
page = self.templatepage('das_faq',
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
@expose
def help(self, *args, **kwargs):
"""
represent DAS help
"""
page = self.templatepage('das_help')
return self.page(page, response_div=False)
@expose
def cli(self, *args, **kwargs):
"""
Serve DAS CLI file download.
"""
clifile = os.path.join(os.environ['DAS_ROOT'],
'src/python/DAS/tools/das_cache_client.py')
return serve_file(clifile, content_type='text/plain')
@expose
def services(self, *args, **kwargs):
"""
represent DAS services
"""
dasdict = {}
daskeys = []
for system, keys in self.dasmgr.mapping.daskeys().items():
tmpdict = {}
for key in keys:
tmpdict[key] = self.dasmgr.mapping.lookup_keys(system, key)
if key not in daskeys:
daskeys.append(key)
dasdict[system] = dict(keys=dict(tmpdict),
apis=self.dasmgr.mapping.list_apis(system))
mapreduce = [r for r in self.dasmgr.rawcache.get_map_reduce()]
page = self.templatepage('das_services', dasdict=dasdict,
daskeys=daskeys, mapreduce=mapreduce)
return self.page(page, response_div=False)
@expose
def api(self, name, **kwargs):
"""
Return DAS mapping record about provided API.
"""
record = self.dasmgr.mapping.api_info(name)
show = kwargs.get('show', 'json')
page = "<b>DAS mapping record</b>"
if show == 'json':
jsoncode = {'jsoncode': json2html(record, "")}
page += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(record, indent=1, width=100)
page += self.templatepage('das_code', code=code)
else:
code = yaml.dump(record, width=100, indent=4,
default_flow_style=False)
page += self.templatepage('das_code', code=code)
return self.page(page, response_div=False)
@expose
def default(self, *args, **kwargs):
"""
Default method.
"""
return self.index(args, kwargs)
def check_input(self, uinput):
"""
Check provided input for valid DAS keys.
"""
error = self.templatepage('das_ambiguous',
input=uinput, entities=', '.join(self.daskeys))
if not uinput:
return error
# check provided input. If at least one word is not part of das_keys
# return ambiguous template.
mongo_query = self.dasmgr.mongoparser.parse(uinput)
fields = mongo_query.get('fields', [])
if not fields:
fields = []
spec = mongo_query.get('spec', {})
if not fields+spec.keys():
return error
for word in fields+spec.keys():
found = 0
for key in das_keys:
if word.find(key) != -1:
found = 1
if not found:
return error
return
@expose
def index(self, *args, **kwargs):
"""
represents DAS web interface.
It uses das_searchform template for
input form and yui_table for output Table widget.
"""
try:
if not args and not kwargs:
# msg = self.templatepage('das_help',
# services = ', '.join(self.dasmgr.keys()),
# keywords = ', '.join(self.dasmgr.das_keys()),
# operators = ', '.join(das_operators()),
# aggregators = ', '.join(das_aggregators()),
# filters = ', '.join(das_filters())
# )
page = self.form()
return self.page(page)
uinput = getarg(kwargs, 'input', '')
results = self.check_input(uinput)
if results:
return self.page(self.form() + results)
view = getarg(kwargs, 'view', 'list')
if args:
return getattr(self, args[0][0])(args[1])
if view not in self.pageviews:
raise Exception("Page view '%s' is not supported" % view)
return getattr(self, '%sview' % view)(kwargs)
except:
return self.error(self.gen_error_msg(kwargs))
@expose
def form(self, uinput=None, msg=None):
"""
provide input DAS search form
"""
page = self.templatepage('das_searchform', input=uinput, msg=msg,
base=self.base)
return page
def gen_error_msg(self, kwargs):
"""
Generate standard error message.
"""
self.daslogger.error(traceback.format_exc())
error = "My request to DAS is failed\n\n"
error += "Input parameters:\n"
for key, val in kwargs.items():
error += '%s: %s\n' % (key, val)
error += "Exception type: %s\nException value: %s\nTime: %s" \
% (sys.exc_info()[0], sys.exc_info()[1], web_time())
error = error.replace("<", "").replace(">", "")
return error
@expose
def error(self, msg):
"""
Show error message.
"""
error = self.templatepage('das_error', msg=msg)
page = self.page(self.form() + error)
return page
@exposedasjson
def wrap2dasjson(self, data):
"""DAS JSON wrapper"""
return data
@exposedasplist
def wrap2dasxml(self, data):
"""DAS XML wrapper"""
return data
@expose
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ''
if args:
recordid = args[0]
spec = {'_id':recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and '_id' in kwargs:
spec = {'_id': kwargs['_id']}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
coll = getarg(kwargs, 'collection', 'merge')
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {'query':json.dumps(query), 'idx':idx, 'limit':limit,
'collection':coll}
path = '/rest/records'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url+path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status':'fail', 'reason':traceback.format_exc()}
res = ""
if result['status'] == 'success':
if recordid: # we got id
for row in result['data']:
if show == 'json':
jsoncode = {'jsoncode': json2html(row, "")}
res += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(row, indent=1, width=100)
res += self.templatepage('das_code', code=code)
else:
code = yaml.dump(row, width=100, indent=4,
default_flow_style=False)
res += self.templatepage('das_code', code=code)
else:
for row in result['data']:
rid = row['_id']
del row['_id']
record = dict(id=rid, daskeys=', '.join(row))
res += self.templatepage('das_record', **record)
else:
res = result['status']
if 'reason' in res:
return self.error(res['reason'])
else:
msg = 'Uknown error, kwargs=' % kwargs
return self.error(msg)
if recordid:
if format:
if format == 'xml':
return self.wrap2dasxml(result['data'])
elif format == 'json':
return self.wrap2dasjson(result['data'])
else:
return self.error('Unsupported data format %s' % format)
page = res
else:
url = '/das/records?'
idict = dict(nrows=nresults, idx=idx,
limit=limit, results=res, url=url)
page = self.templatepage('das_pagination', **idict)
form = self.form(uinput="")
ctime = (time.time()-time0)
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def nresults(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
params = {'query':uinput}
path = '/rest/nresults'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url+path, params, headers=headers)
record = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
record = {'status':'fail', 'reason':traceback.format_exc()}
if record['status'] == 'success':
return record['nresults']
else:
msg = "nresults returns status: %s" % str(record)
self.daslogger.info(msg)
return -1
def send_request(self, method, kwargs):
"Send POST request to server with provided parameters"
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
format = getarg(kwargs, 'format', '')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
skey = getarg(kwargs, 'sort', '')
sdir = getarg(kwargs, 'dir', 'asc')
params = {'query':uinput, 'idx':idx, 'limit':limit,
'skey':skey, 'order':sdir}
if method == 'POST':
path = '/rest/create'
elif method == 'GET':
path = '/rest/request'
else:
raise Exception('Unsupported method %s' % method)
headers = {'Accept': 'application/json',
'Content-type': 'application/json'}
try:
data = urllib2_request(method, url+path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status':'fail', 'reason':traceback.format_exc()}
return result
def result(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
result = self.send_request('GET', kwargs)
res = []
if type(result) is bytes:
data = json.loads(result)
else:
data = result
if data['status'] == 'success':
res = data['data']
return res
@exposedasplist
def xmlview(self, kwargs):
"""
provide DAS XML
"""
rows = self.result(kwargs)
return rows
@exposedasjson
def jsonview(self, kwargs):
"""
provide DAS JSON
"""
rows = self.result(kwargs)
return rows
def convert2ui(self, idict):
"""
Convert input row (dict) into UI presentation
"""
for key in idict.keys():
if key == 'das' or key == '_id' or key == 'das_id':
continue
for item in self.dasmapping.presentation(key):
try:
daskey = item['das']
uikey = item['ui']
for value in access(idict, daskey):
yield uikey, value
except:
yield key, idict[key]
@expose
def listview(self, kwargs):
"""
provide DAS list view
"""
# force to load the page all the time
cherrypy.response.headers['Cache-Control'] = 'no-cache'
cherrypy.response.headers['Pragma'] = 'no-cache'
time0 = time.time()
ajaxreq = getarg(kwargs, 'ajax', 0)
uinput = getarg(kwargs, 'input', '')
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
form = self.form(uinput=uinput)
# self.status sends request to Cache Server
# Cache Server uses das_core to retrieve status
status = self.status(input=uinput, ajax=0)
if status == 'no data':
# no data in raw cache, send POST request
self.send_request('POST', kwargs)
ctime = (time.time()-time0)
# page = self.templatepage('not_ready')
page = self.status(input=uinput)
page = self.page(form + page, ctime=ctime)
return page
elif status == 'fail':
kwargs['reason'] = 'Unable to get status from data-service'
return self.error(self.gen_error_msg(kwargs))
total = self.nresults(kwargs)
rows = self.result(kwargs)
nrows = len(rows)
page = ""
ndict = {'nrows':total, 'limit':limit}
page = self.templatepage('das_nrecords', **ndict)
# for nrecord in range(0, len(rows)):
# row = rows[nrecord]
# style = "white"
# if nrecord % 2:
# style = "white"
# else:
# style = "gray"
style = "white"
for row in rows:
id = row['_id']
page += '<div class="%s"><hr class="line" />' % style
gen = self.convert2ui(row)
for uikey, value in [k for k, g in groupby(gen)]:
page += "<b>%s</b>: %s<br />" % (uikey, value)
pad = ""
if show == 'json':
jsoncode = {'jsoncode': json2html(row, pad)}
jsonhtml = self.templatepage('das_json', **jsoncode)
jsondict = dict(data=jsonhtml, id=id, rec_id=id)
page += self.templatepage('das_row', **jsondict)
elif show == 'code':
code = pformat(row, indent=1, width=100)
data = self.templatepage('das_code', code=code)
datadict = {'data':data, 'id':id, rec_id:id}
page += self.templatepage('das_row', **datadict)
else:
code = yaml.dump(row, width=100, indent=4,
default_flow_style=False)
data = self.templatepage('das_code', code=code)
datadict = {'data':data, 'id':id, rec_id:id}
page += self.templatepage('das_row', **datadict)
page += '</div>'
ctime = (time.time()-time0)
return self.page(form + page, ctime=ctime)
@exposetext
def plainview(self, kwargs):
"""
provide DAS plain view
"""
rows, total, form = self.result(kwargs)
page = ""
for item in rows:
item = str(item).replace('[','').replace(']','')
page += "%s\n" % item.replace("'","")
return page
@exposejson
def yuijson(self, **kwargs):
"""
Provide JSON in YUI compatible format to be used in DynamicData table
widget, see
http://developer.yahoo.com/yui/examples/datatable/dt_dynamicdata.html
"""
rows = self.result(kwargs)
rowlist = []
id = 0
for row in rows:
das = row['das']
if type(das) is dict:
das = [das]
resdict = {}
for jdx in range(0, len(das)):
item = das[jdx]
resdict[id] = id
for idx in range(0, len(item['system'])):
api = item['api'][idx]
system = item['system'][idx]
key = item['selection_keys'][idx]
data = row[key]
if type(data) is list:
data = data[jdx]
if type(data) is list:
data = data[idx]
# I need to extract from DAS object the values for UI keys
for item in self.dasmapping.presentation(key):
daskey = item['das']
uiname = item['ui']
if uiname not in resdict:
resdict[uiname] = ""
# look at key attributes, which may be compound as well
# e.g. block.replica.se
if type(data) is dict:
result = dict(data)
elif type(data) is list:
result = list(data)
else:
result = data
res = ""
try:
for elem in daskey.split('.')[1:]:
if elem in result:
res = result[elem]
resdict[uiname] = res
except:
pass
# pad = ""
# jsoncode = {'jsoncode': json2html(data, pad)}
# jsonhtml = self.templatepage('das_json', **jsoncode)
# jsondict = {'id':id, 'system':system, 'api':api, key:jsonhtml}
if resdict not in rowlist:
rowlist.append(resdict)
id += 1
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
total = len(rowlist)
jsondict = {'recordsReturned': len(rowlist),
'totalRecords': total, 'startIndex':idx,
'sort':'true', 'dir':'asc',
'pageSize': limit,
'records': rowlist}
return jsondict
@expose
def tableview(self, kwargs):
"""
provide DAS table view
"""
kwargs['format'] = 'html'
uinput = getarg(kwargs, 'input', '')
ajaxreq = getarg(kwargs, 'ajax', 0)
form = self.form(uinput=uinput)
time0 = time.time()
total = self.nresults(kwargs)
if not total:
ctime = (time.time()-time0)
form = self.form(uinput)
page = self.templatepage('not_ready')
page = self.page(form + page, ctime=ctime)
return page
# find out which selection keys were used
selkeys = uinput.replace('find ', '').split(' where ')[0].split(',')
uikeys = []
for key in selkeys:
res = self.dasmapping.presentation(key)
uikeys += [item['ui'] for item in res]
titles = ["id"] + uikeys
coldefs = ""
for title in titles:
coldefs += '{key:"%s",label:"%s",sortable:true,resizeable:true},' \
% (title, title)
coldefs = "[%s]" % coldefs[:-1] # remove last comma
coldefs = coldefs.replace("},{","},\n{")
limit = getarg(kwargs, 'limit', 10)
names = {'titlelist':titles,
'coldefs':coldefs, 'rowsperpage':limit,
'total':total, 'tag':'mytag', 'ajax':ajaxreq,
'input':urllib.urlencode(dict(input=uinput))}
page = self.templatepage('das_table', **names)
ctime = (time.time()-time0)
page = self.page(form + page, ctime=ctime)
return page
@expose
def status(self, **kwargs):
"""
Place request to obtain status about given query
"""
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
req = """
<script type="application/javascript">
setTimeout('ajaxStatus()',3000)
</script>"""
def set_header():
"Set HTTP header parameters"
tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
cherrypy.response.headers['Expire'] = tstamp
cherrypy.response.headers['Cache-control'] = 'no-cache'
uinput = kwargs.get('input', '')
uinput = urllib.unquote_plus(uinput)
ajax = kwargs.get('ajax', 1)
view = kwargs.get('view', 'list')
params = {'query':uinput}
path = '/rest/status'
url = self.cachesrv
headers = {'Accept': 'application/json'}
try:
res = urllib2_request('GET', url+path, params, headers=headers)
data = json.loads(res)
except:
self.daslogger.error(traceback.format_exc())
data = {'status':'fail'}
if ajax:
cherrypy.response.headers['Content-Type'] = 'text/xml'
if data['status'] == 'ok':
page = '<script type="application/javascript">reload()</script>'
elif data['status'] == 'fail':
page = '<script type="application/javascript">reload()</script>'
page += self.error(self.gen_error_msg(kwargs))
else:
page = img + ' ' + str(data['status']) + ', please wait...'
img_stop = ''
page += ', <a href="/das/">stop</a> request'
page += req
set_header()
page = ajax_response(page)
else:
try:
page = data['status']
except:
page = traceback.format_exc()
return page
def main():
"Main function"
optmgr = DASOptionParser()
opts = optmgr.parser.parse_args()
t0 = time.time()
query = opts.query
if 'instance' not in query:
query = ' instance=prod/global ' + query
debug = opts.verbose
dascore = DASCore(debug=debug, nores=opts.noresults)
if opts.hash:
dasquery = DASQuery(query)
mongo_query = dasquery.mongo_query
service_map = dasquery.service_apis_map()
str_query = dasquery.storage_query
print("---------------")
print("DAS-QL query :", query)
print("DAS query :", dasquery)
print("Mongo query :", mongo_query)
print("Storage query :", str_query)
print("Services :\n")
for srv, val in service_map.items():
print("%s : %s\n" % (srv, ', '.join(val)))
sys.exit(0)
sdict = dascore.keys()
if opts.services:
msg = "DAS services:"
print(msg)
print("-"*len(msg))
keys = list(sdict.keys())
keys.sort()
for key in keys:
print(key)
elif opts.service:
msg = "DAS service %s:" % opts.service
print(msg)
print("-"*len(msg))
keys = sdict[opts.service]
keys.sort()
for key in keys:
print(key)
elif opts.jsfile:
kws_js(dascore, query, opts.idx, opts.limit, opts.jsfile, debug)
sys.exit(0)
elif opts.kfile:
keylearning_js(dascore, query, opts.kfile, debug)
sys.exit(0)
elif query:
idx = opts.idx
limit = opts.limit
output = opts.nooutput
plain = opts.plain
qcache = opts.qcache
if opts.profile:
import cProfile # python profiler
import pstats # profiler statistics
cmd = 'run(dascore,query,idx,limit,output,plain)'
cProfile.runctx(cmd, globals(), locals(), 'profile.dat')
info = pstats.Stats('profile.dat')
info.sort_stats('cumulative')
info.print_stats()
else:
run(dascore, query, idx, limit, output, plain)
elif opts.dasconfig:
print(pformat(dascore.dasconfig))
else:
print()
print("DAS CLI interface, no actions found,")
print("please use --help for more options.")
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
timer = get_das_timer()
print("\nDAS execution time:\n")
if debug:
timelist = []
for _, timerdict in timer.items():
counter = timerdict['counter']
tag = timerdict['tag']
exetime = timerdict['time']
timelist.append((counter, tag, exetime))
timelist.sort()
for _, tag, exetime in timelist:
print("%s %s sec" % (tag, round(exetime, 2)))
print("Total %s sec, %s" % (round(time.time()-t0, 2), timestamp))
def main():
"Main function"
optmgr = DASOptionParser()
opts, _ = optmgr.getOpt()
t0 = time.time()
query = opts.query
if 'instance' not in query:
query += ' instance=cms_dbs_prod_global'
debug = opts.verbose
dascore = DASCore(debug=debug, nores=opts.noresults)
if opts.hash:
dasquery = DASQuery(query)
mongo_query = dasquery.mongo_query
service_map = dasquery.service_apis_map()
str_query = dasquery.storage_query
print "---------------"
print "DAS-QL query :", query
print "DAS query :", dasquery
print "Mongo query :", mongo_query
print "Storage query :", str_query
print "Services :\n"
for srv, val in service_map.items():
print "%s : %s\n" % (srv, ', '.join(val))
sys.exit(0)
sdict = dascore.keys()
if opts.services:
msg = "DAS services:"
print msg
print "-"*len(msg)
keys = sdict.keys()
keys.sort()
for key in keys:
print key
elif opts.service:
msg = "DAS service %s:" % opts.service
print msg
print "-"*len(msg)
keys = sdict[opts.service]
keys.sort()
for key in keys:
print key
elif query:
idx = opts.idx
limit = opts.limit
output = opts.nooutput
plain = opts.plain
if opts.profile:
import cProfile # python profiler
import pstats # profiler statistics
cmd = 'run(dascore,query,idx,limit,output,plain)'
cProfile.runctx(cmd, globals(), locals(), 'profile.dat')
info = pstats.Stats('profile.dat')
info.sort_stats('cumulative')
info.print_stats()
else:
run(dascore, query, idx, limit, output, plain)
elif opts.dasconfig:
print pformat(dascore.dasconfig)
else:
print
print "DAS CLI interface, no actions found,"
print "please use --help for more options."
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
timer = get_das_timer()
print "\nDAS execution time:\n"
if debug:
timelist = []
for _, timerdict in timer.items():
counter = timerdict['counter']
tag = timerdict['tag']
exetime = timerdict['time']
timelist.append((counter, tag, exetime))
timelist.sort()
for _, tag, exetime in timelist:
print "%s %s sec" % (tag, round(exetime, 2))
print "Total %s sec, %s" % (round(time.time()-t0, 2), timestamp)
class DASSearch(DASWebManager):
"""
DAS web interface.
"""
def __init__(self, config={}):
DASWebManager.__init__(self, config)
try:
# try what is supplied from WebTools framework
cdict = self.config.dictionary_()
self.cachesrv = cdict.get('cache_server_url',
'http://localhost:8211')
self.base = '/dascontrollers'
except:
# stand-alone version
self.cachesrv = config.get('cache_server_url',
'http://localhost:8211')
self.base = '/das'
self.dasmgr = DASCore()
self.daskeys = self.dasmgr.das_keys()
self.daskeys.sort()
self.dasmapping = self.dasmgr.mapping
self.daslogger = self.dasmgr.logger
self.pageviews = ['xml', 'list', 'json', 'yuijson']
msg = "DASSearch::init is started with base=%s" % self.base
self.daslogger.debug(msg)
print(msg)
def top(self):
"""
Define masthead for all DAS web pages
"""
return self.templatepage('das_top', base=self.base)
def bottom(self, response_div=True):
"""
Define footer for all DAS web pages
"""
return self.templatepage('das_bottom', div=response_div)
def page(self, content, ctime=None, response_div=True):
"""
Define footer for all DAS web pages
"""
page = self.top()
page += content
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
services = self.dasmgr.keys()
srv = ""
for key in services.keys():
srv += "%s, " % key
srv = srv[:-2] # remove last comma
page += self.templatepage('das_bottom',
ctime=ctime,
services=srv,
timestamp=timestamp,
div=response_div)
return page
@expose
def faq(self, *args, **kwargs):
"""
represent DAS FAQ.
"""
page = self.templatepage('das_faq',
operators=', '.join(das_operators()),
aggregators=', '.join(das_aggregators()))
return self.page(page, response_div=False)
@expose
def help(self, *args, **kwargs):
"""
represent DAS help
"""
page = self.templatepage('das_help')
return self.page(page, response_div=False)
@expose
def cli(self, *args, **kwargs):
"""
Serve DAS CLI file download.
"""
clifile = os.path.join(os.environ['DAS_ROOT'],
'src/python/DAS/tools/das_cache_client.py')
return serve_file(clifile, content_type='text/plain')
@expose
def services(self, *args, **kwargs):
"""
represent DAS services
"""
dasdict = {}
daskeys = []
for system, keys in self.dasmgr.mapping.daskeys().items():
tmpdict = {}
for key in keys:
tmpdict[key] = self.dasmgr.mapping.lookup_keys(system, key)
if key not in daskeys:
daskeys.append(key)
dasdict[system] = dict(keys=dict(tmpdict),
apis=self.dasmgr.mapping.list_apis(system))
mapreduce = [r for r in self.dasmgr.rawcache.get_map_reduce()]
page = self.templatepage('das_services',
dasdict=dasdict,
daskeys=daskeys,
mapreduce=mapreduce)
return self.page(page, response_div=False)
@expose
def api(self, name, **kwargs):
"""
Return DAS mapping record about provided API.
"""
record = self.dasmgr.mapping.api_info(name)
show = kwargs.get('show', 'json')
page = "<b>DAS mapping record</b>"
if show == 'json':
jsoncode = {'jsoncode': json2html(record, "")}
page += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(record, indent=1, width=100)
page += self.templatepage('das_code', code=code)
else:
code = yaml.dump(record,
width=100,
indent=4,
default_flow_style=False)
page += self.templatepage('das_code', code=code)
return self.page(page, response_div=False)
@expose
def default(self, *args, **kwargs):
"""
Default method.
"""
return self.index(args, kwargs)
def check_input(self, uinput):
"""
Check provided input for valid DAS keys.
"""
error = self.templatepage('das_ambiguous',
input=uinput,
entities=', '.join(self.daskeys))
if not uinput:
return error
# check provided input. If at least one word is not part of das_keys
# return ambiguous template.
mongo_query = self.dasmgr.mongoparser.parse(uinput)
fields = mongo_query.get('fields', [])
if not fields:
fields = []
spec = mongo_query.get('spec', {})
if not fields + spec.keys():
return error
for word in fields + spec.keys():
found = 0
for key in das_keys:
if word.find(key) != -1:
found = 1
if not found:
return error
return
@expose
def index(self, *args, **kwargs):
"""
represents DAS web interface.
It uses das_searchform template for
input form and yui_table for output Table widget.
"""
try:
if not args and not kwargs:
# msg = self.templatepage('das_help',
# services = ', '.join(self.dasmgr.keys()),
# keywords = ', '.join(self.dasmgr.das_keys()),
# operators = ', '.join(das_operators()),
# aggregators = ', '.join(das_aggregators()),
# filters = ', '.join(das_filters())
# )
page = self.form()
return self.page(page)
uinput = getarg(kwargs, 'input', '')
results = self.check_input(uinput)
if results:
return self.page(self.form() + results)
view = getarg(kwargs, 'view', 'list')
if args:
return getattr(self, args[0][0])(args[1])
if view not in self.pageviews:
raise Exception("Page view '%s' is not supported" % view)
return getattr(self, '%sview' % view)(kwargs)
except:
return self.error(self.gen_error_msg(kwargs))
@expose
def form(self, uinput=None, msg=None):
"""
provide input DAS search form
"""
page = self.templatepage('das_searchform',
input=uinput,
msg=msg,
base=self.base)
return page
def gen_error_msg(self, kwargs):
"""
Generate standard error message.
"""
self.daslogger.error(traceback.format_exc())
error = "My request to DAS is failed\n\n"
error += "Input parameters:\n"
for key, val in kwargs.items():
error += '%s: %s\n' % (key, val)
error += "Exception type: %s\nException value: %s\nTime: %s" \
% (sys.exc_info()[0], sys.exc_info()[1], web_time())
error = error.replace("<", "").replace(">", "")
return error
@expose
def error(self, msg):
"""
Show error message.
"""
error = self.templatepage('das_error', msg=msg)
page = self.page(self.form() + error)
return page
@exposedasjson
def wrap2dasjson(self, data):
"""DAS JSON wrapper"""
return data
@exposedasplist
def wrap2dasxml(self, data):
"""DAS XML wrapper"""
return data
@expose
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
format = ''
if args:
recordid = args[0]
spec = {'_id': recordid}
fields = None
query = dict(fields=fields, spec=spec)
if len(args) == 2:
format = args[1]
elif kwargs and '_id' in kwargs:
spec = {'_id': kwargs['_id']}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
nresults = self.nresults(query)
time0 = time.time()
url = self.cachesrv
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
coll = getarg(kwargs, 'collection', 'merge')
# params = {'query':json.dumps(query), 'idx':idx, 'limit':limit}
# path = '/rest/request'
params = {
'query': json.dumps(query),
'idx': idx,
'limit': limit,
'collection': coll
}
path = '/rest/records'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET',
url + path,
params,
headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status': 'fail', 'reason': traceback.format_exc()}
res = ""
if result['status'] == 'success':
if recordid: # we got id
for row in result['data']:
if show == 'json':
jsoncode = {'jsoncode': json2html(row, "")}
res += self.templatepage('das_json', **jsoncode)
elif show == 'code':
code = pformat(row, indent=1, width=100)
res += self.templatepage('das_code', code=code)
else:
code = yaml.dump(row,
width=100,
indent=4,
default_flow_style=False)
res += self.templatepage('das_code', code=code)
else:
for row in result['data']:
rid = row['_id']
del row['_id']
record = dict(id=rid, daskeys=', '.join(row))
res += self.templatepage('das_record', **record)
else:
res = result['status']
if 'reason' in res:
return self.error(res['reason'])
else:
msg = 'Uknown error, kwargs=' % kwargs
return self.error(msg)
if recordid:
if format:
if format == 'xml':
return self.wrap2dasxml(result['data'])
elif format == 'json':
return self.wrap2dasjson(result['data'])
else:
return self.error('Unsupported data format %s' %
format)
page = res
else:
url = '/das/records?'
idict = dict(nrows=nresults,
idx=idx,
limit=limit,
results=res,
url=url)
page = self.templatepage('das_pagination', **idict)
form = self.form(uinput="")
ctime = (time.time() - time0)
page = self.page(form + page, ctime=ctime)
return page
except:
return self.error(self.gen_error_msg(kwargs))
def nresults(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
params = {'query': uinput}
path = '/rest/nresults'
headers = {"Accept": "application/json"}
try:
data = urllib2_request('GET', url + path, params, headers=headers)
record = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
record = {'status': 'fail', 'reason': traceback.format_exc()}
if record['status'] == 'success':
return record['nresults']
else:
msg = "nresults returns status: %s" % str(record)
self.daslogger.info(msg)
return -1
def send_request(self, method, kwargs):
"Send POST request to server with provided parameters"
url = self.cachesrv
uinput = getarg(kwargs, 'input', '')
format = getarg(kwargs, 'format', '')
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
skey = getarg(kwargs, 'sort', '')
sdir = getarg(kwargs, 'dir', 'asc')
params = {
'query': uinput,
'idx': idx,
'limit': limit,
'skey': skey,
'order': sdir
}
if method == 'POST':
path = '/rest/create'
elif method == 'GET':
path = '/rest/request'
else:
raise Exception('Unsupported method %s' % method)
headers = {
'Accept': 'application/json',
'Content-type': 'application/json'
}
try:
data = urllib2_request(method, url + path, params, headers=headers)
result = json.loads(data)
except:
self.daslogger.error(traceback.format_exc())
result = {'status': 'fail', 'reason': traceback.format_exc()}
return result
def result(self, kwargs):
"""
invoke DAS search call, parse results and return them to
web methods
"""
result = self.send_request('GET', kwargs)
res = []
if type(result) is bytes:
data = json.loads(result)
else:
data = result
if data['status'] == 'success':
res = data['data']
return res
@exposedasplist
def xmlview(self, kwargs):
"""
provide DAS XML
"""
rows = self.result(kwargs)
return rows
@exposedasjson
def jsonview(self, kwargs):
"""
provide DAS JSON
"""
rows = self.result(kwargs)
return rows
def convert2ui(self, idict):
"""
Convert input row (dict) into UI presentation
"""
for key in idict.keys():
if key == 'das' or key == '_id' or key == 'das_id':
continue
for item in self.dasmapping.presentation(key):
try:
daskey = item['das']
uikey = item['ui']
for value in access(idict, daskey):
yield uikey, value
except:
yield key, idict[key]
@expose
def listview(self, kwargs):
"""
provide DAS list view
"""
# force to load the page all the time
cherrypy.response.headers['Cache-Control'] = 'no-cache'
cherrypy.response.headers['Pragma'] = 'no-cache'
time0 = time.time()
ajaxreq = getarg(kwargs, 'ajax', 0)
uinput = getarg(kwargs, 'input', '')
limit = getarg(kwargs, 'limit', 10)
show = getarg(kwargs, 'show', 'json')
form = self.form(uinput=uinput)
# self.status sends request to Cache Server
# Cache Server uses das_core to retrieve status
status = self.status(input=uinput, ajax=0)
if status == 'no data':
# no data in raw cache, send POST request
self.send_request('POST', kwargs)
ctime = (time.time() - time0)
# page = self.templatepage('not_ready')
page = self.status(input=uinput)
page = self.page(form + page, ctime=ctime)
return page
elif status == 'fail':
kwargs['reason'] = 'Unable to get status from data-service'
return self.error(self.gen_error_msg(kwargs))
total = self.nresults(kwargs)
rows = self.result(kwargs)
nrows = len(rows)
page = ""
ndict = {'nrows': total, 'limit': limit}
page = self.templatepage('das_nrecords', **ndict)
# for nrecord in range(0, len(rows)):
# row = rows[nrecord]
# style = "white"
# if nrecord % 2:
# style = "white"
# else:
# style = "gray"
style = "white"
for row in rows:
id = row['_id']
page += '<div class="%s"><hr class="line" />' % style
gen = self.convert2ui(row)
for uikey, value in [k for k, g in groupby(gen)]:
page += "<b>%s</b>: %s<br />" % (uikey, value)
pad = ""
if show == 'json':
jsoncode = {'jsoncode': json2html(row, pad)}
jsonhtml = self.templatepage('das_json', **jsoncode)
jsondict = dict(data=jsonhtml, id=id, rec_id=id)
page += self.templatepage('das_row', **jsondict)
elif show == 'code':
code = pformat(row, indent=1, width=100)
data = self.templatepage('das_code', code=code)
datadict = {'data': data, 'id': id, rec_id: id}
page += self.templatepage('das_row', **datadict)
else:
code = yaml.dump(row,
width=100,
indent=4,
default_flow_style=False)
data = self.templatepage('das_code', code=code)
datadict = {'data': data, 'id': id, rec_id: id}
page += self.templatepage('das_row', **datadict)
page += '</div>'
ctime = (time.time() - time0)
return self.page(form + page, ctime=ctime)
@exposetext
def plainview(self, kwargs):
"""
provide DAS plain view
"""
rows, total, form = self.result(kwargs)
page = ""
for item in rows:
item = str(item).replace('[', '').replace(']', '')
page += "%s\n" % item.replace("'", "")
return page
@exposejson
def yuijson(self, **kwargs):
"""
Provide JSON in YUI compatible format to be used in DynamicData table
widget, see
http://developer.yahoo.com/yui/examples/datatable/dt_dynamicdata.html
"""
rows = self.result(kwargs)
rowlist = []
id = 0
for row in rows:
das = row['das']
if type(das) is dict:
das = [das]
resdict = {}
for jdx in range(0, len(das)):
item = das[jdx]
resdict[id] = id
for idx in range(0, len(item['system'])):
api = item['api'][idx]
system = item['system'][idx]
key = item['selection_keys'][idx]
data = row[key]
if type(data) is list:
data = data[jdx]
if type(data) is list:
data = data[idx]
# I need to extract from DAS object the values for UI keys
for item in self.dasmapping.presentation(key):
daskey = item['das']
uiname = item['ui']
if uiname not in resdict:
resdict[uiname] = ""
# look at key attributes, which may be compound as well
# e.g. block.replica.se
if type(data) is dict:
result = dict(data)
elif type(data) is list:
result = list(data)
else:
result = data
res = ""
try:
for elem in daskey.split('.')[1:]:
if elem in result:
res = result[elem]
resdict[uiname] = res
except:
pass
# pad = ""
# jsoncode = {'jsoncode': json2html(data, pad)}
# jsonhtml = self.templatepage('das_json', **jsoncode)
# jsondict = {'id':id, 'system':system, 'api':api, key:jsonhtml}
if resdict not in rowlist:
rowlist.append(resdict)
id += 1
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
total = len(rowlist)
jsondict = {
'recordsReturned': len(rowlist),
'totalRecords': total,
'startIndex': idx,
'sort': 'true',
'dir': 'asc',
'pageSize': limit,
'records': rowlist
}
return jsondict
@expose
def tableview(self, kwargs):
"""
provide DAS table view
"""
kwargs['format'] = 'html'
uinput = getarg(kwargs, 'input', '')
ajaxreq = getarg(kwargs, 'ajax', 0)
form = self.form(uinput=uinput)
time0 = time.time()
total = self.nresults(kwargs)
if not total:
ctime = (time.time() - time0)
form = self.form(uinput)
page = self.templatepage('not_ready')
page = self.page(form + page, ctime=ctime)
return page
# find out which selection keys were used
selkeys = uinput.replace('find ', '').split(' where ')[0].split(',')
uikeys = []
for key in selkeys:
res = self.dasmapping.presentation(key)
uikeys += [item['ui'] for item in res]
titles = ["id"] + uikeys
coldefs = ""
for title in titles:
coldefs += '{key:"%s",label:"%s",sortable:true,resizeable:true},' \
% (title, title)
coldefs = "[%s]" % coldefs[:-1] # remove last comma
coldefs = coldefs.replace("},{", "},\n{")
limit = getarg(kwargs, 'limit', 10)
names = {
'titlelist': titles,
'coldefs': coldefs,
'rowsperpage': limit,
'total': total,
'tag': 'mytag',
'ajax': ajaxreq,
'input': urllib.urlencode(dict(input=uinput))
}
page = self.templatepage('das_table', **names)
ctime = (time.time() - time0)
page = self.page(form + page, ctime=ctime)
return page
@expose
def status(self, **kwargs):
"""
Place request to obtain status about given query
"""
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
req = """
<script type="application/javascript">
setTimeout('ajaxStatus()',3000)
</script>"""
def set_header():
"Set HTTP header parameters"
tstamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
cherrypy.response.headers['Expire'] = tstamp
cherrypy.response.headers['Cache-control'] = 'no-cache'
uinput = kwargs.get('input', '')
uinput = urllib.unquote_plus(uinput)
ajax = kwargs.get('ajax', 1)
view = kwargs.get('view', 'list')
params = {'query': uinput}
path = '/rest/status'
url = self.cachesrv
headers = {'Accept': 'application/json'}
try:
res = urllib2_request('GET', url + path, params, headers=headers)
data = json.loads(res)
except:
self.daslogger.error(traceback.format_exc())
data = {'status': 'fail'}
if ajax:
cherrypy.response.headers['Content-Type'] = 'text/xml'
if data['status'] == 'ok':
page = '<script type="application/javascript">reload()</script>'
elif data['status'] == 'fail':
page = '<script type="application/javascript">reload()</script>'
page += self.error(self.gen_error_msg(kwargs))
else:
page = img + ' ' + str(data['status']) + ', please wait...'
img_stop = ''
page += ', <a href="/das/">stop</a> request'
page += req
set_header()
page = ajax_response(page)
else:
try:
page = data['status']
except:
page = traceback.format_exc()
return page
