def api(self, name, **kwargs):
"""
Return DAS mapping record about provided API.
"""
record = self.dasmgr.mapping.api_info(name)
page = "<b>DAS mapping record</b>"
page += das_json(record)
return self.page(page, response_div=False)
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
if args:
recordid = args[0]
spec = {'_id':ObjectId(recordid)}
fields = None
query = dict(fields=fields, spec=spec)
elif kwargs and kwargs.has_key('_id'):
spec = {'_id': ObjectId(kwargs['_id'])}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
res = ''
time0 = time.time()
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 10)
coll = kwargs.get('collection', 'merge')
inst = kwargs.get('instance', self.dbs_global)
form = self.form(uinput="")
check, content = self.generate_dasquery(query, inst)
if check:
return self.page(form + content, ctime=time.time()-time0)
dasquery = content # returned content is valid DAS query
nresults = self.dasmgr.rawcache.nresults(dasquery, coll)
gen = self.dasmgr.rawcache.get_from_cache\
(dasquery, idx=idx, limit=limit, collection=coll)
if recordid: # we got id
for row in gen:
res += das_json(row)
else:
for row in gen:
rid = row['_id']
del row['_id']
res += self.templatepage('das_record', \
id=rid, collection=coll, daskeys=', '.join(row))
if recordid:
page = res
else:
url = '/das/records?'
if nresults:
page = self.templatepage('das_pagination', \
nrows=nresults, idx=idx, limit=limit, url=url)
else:
page = 'No results found, nresults=%s' % nresults
page += res
ctime = (time.time()-time0)
page = self.page(form + page, ctime=ctime)
return page
except Exception as exc:
print_exc(exc)
return self.error(gen_error_msg(kwargs))
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
total = head.get('nresults', 0)
incache = head.get('incache')
main = self.pagination(total, incache, kwargs)
style = 'white'
page = ''
pad = ''
for row in data:
if not row:
continue
if style == 'white':
style = 'gray'
else:
style = 'white'
try:
mongo_id = row['_id']
except Exception as exc:
msg = str(exc)
msg += '\nFail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
jsonhtml = das_json(row, pad)
if row.has_key('das') and row['das'].has_key('conflict'):
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
page += self.templatepage('das_row', systems='', \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict)
page += '</div>'
main += page
main += '<div align="right">DAS cache server time: %5.3f sec</div>' \
% head['ctime']
return main
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
aggrtrs = dasquery.aggregators
main = self.pagination(head)
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
pval = None
lkey = None
if 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(row)
try:
lkey = pkey.split('.')[0]
if pkey == 'summary':
pval = row[pkey]
else:
pval = [i for i in DotDict(row).get_values(pkey)]
if isinstance(pval, list):
if pval and not isinstance(pval[0], list):
pval = list(set(pval))
else:
pval = list(set(pval))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
if isinstance(pval, basestring):
pval = int(pval)
except Exception as exc:
msg = "Fail to extract pval for pkey='%s', lkey='%s'" \
% (pkey, lkey)
msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
print msg
print_exc(exc)
pval = 'N/A'
try:
if not filters:
if pkey == 'summary':
page += 'Summary information:'
elif pval and pval != 'N/A':
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
args = make_args(lkey, pval, inst)
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (args, pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if 'link' in item:
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = '<a href="/das/download?lfn=%s">Download</a>'\
% lfn if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except Exception as exc:
print_exc(exc)
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
if filters and not aggrtrs:
page += add_filter_values(row, filters)
else:
page += adjust_values(func, gen, links, pkey)
pad = ""
try:
if 'das' in row and 'system' in row['das']:
systems = self.systems(row['das']['system'])
else:
systems = "" # no das record
print dastimestamp('DAS ERROR '), \
'record without DAS key', row
except KeyError as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(dasquery, row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if 'das' in row and 'conflict' in row['das']:
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict)
page += '</div>'
old = row
main += fltpage
if dup and not dasquery.aggregators:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
proc_time = self.processing_time(dasquery)
if proc_time:
msg = 'processing time: %5.3f sec, ' % proc_time
else:
msg = ''
msg += 'cache server time: %5.3f sec' % head['ctime']
main += '<div align="right">%s</div>' % msg
return main
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
incache = head.get('incache')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
main = self.pagination(total, incache, kwargs)
if main.find('das_noresults') == -1:
main += self.templatepage('das_colors', colors=self.colors)
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
lkey = None
if row.has_key('das') and row['das'].has_key('primary_key'):
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(row)
try:
lkey = pkey.split('.')[0]
pval = list(set(DotDict(row).get_values(pkey)))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
pval = int(pval)
if pval:
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, pval, inst), pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if item.has_key('link'):
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = self.templatepage(\
'filemover', lfn=lfn) if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except:
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
page += add_filter_values(row, filters)
page += adjust_values(func, gen, links)
pad = ""
try:
systems = self.systems(row['das']['system'])
if row['das']['system'] == ['combined'] or \
row['das']['system'] == [u'combined']:
if lkey:
rowsystems = DotDict(row).get('%s.combined' % lkey)
try:
systems = self.systems(rowsystems)
except TypeError as _err:
systems = self.systems(['combined'])
except KeyError:
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if row.has_key('das') and row['das'].has_key('conflict'):
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict)
page += '</div>'
old = row
main += fltpage
if dup:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
main += '<div align="right">DAS cache server time: %5.3f sec</div>' \
% head['ctime']
return main
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
aggrtrs = dasquery.aggregators
pager = self.pagination(head)
main = pager
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
pval = None
lkey = None
if 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(dasquery)
try:
lkey = pkey.split('.')[0]
if pkey == 'summary':
pval = row[pkey]
else:
pval = [i for i in DotDict(row).get_values(pkey)]
if isinstance(pval, list):
if pval and not isinstance(pval[0], list):
pval = list(set(pval))
else:
pval = list(set(pval))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
if isinstance(pval, basestring):
pval = int(pval)
except Exception as exc:
msg = "Fail to extract pval for pkey='%s', lkey='%s'" \
% (pkey, lkey)
msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
print(msg)
print_exc(exc)
pval = 'N/A'
try:
if not filters:
if pkey == 'summary':
page += 'Summary information:'
elif pval and pval != 'N/A':
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
args = make_args(lkey, pval, inst)
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (args, pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if 'link' in item:
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = '<a href="/das/download?lfn=%s">Download</a>'\
% lfn if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
links.append(self.templatepage(\
'xsecdb', primds=path.split('/')[1]))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except Exception as exc:
print_exc(exc)
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
if filters and not aggrtrs:
page += add_filter_values(row, filters)
else:
page += adjust_values(func, gen, links, pkey)
pad = ""
try:
if 'das' in row and 'system' in row['das']:
systems = self.systems(row['das']['system'])
else:
systems = "" # no das record
print(dastimestamp('DAS ERROR '), \
'record without DAS key', row)
except KeyError as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(dasquery, row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if 'das' in row and 'conflict' in row['das']:
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
hints = ''
for hint in row.get('hints', {}):
if hint:
hints += self.templatepage('hint',
hint=hint, base=self.base, dbs=self.dbs_global)
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict, hints=hints)
page += '</div>'
old = row
main += fltpage
if dup and not dasquery.aggregators:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
if total>10:
main += '<hr class="line" />'
main += pager
main += '<hr class="line" />'
proc_time = self.processing_time(dasquery)
if proc_time:
msg = 'processing time: %5.3f sec, ' % proc_time
else:
msg = ''
msg += 'cache server time: %5.3f sec' % head['ctime']
main += '<div align="right">%s</div>' % msg
return main
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
if args:
recordid = args[0]
spec = {'_id':ObjectId(recordid)}
fields = None
query = dict(fields=fields, spec=spec)
elif kwargs and '_id' in kwargs:
spec = {'_id': ObjectId(kwargs['_id'])}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
res = ''
time0 = time.time()
idx = getarg(kwargs, 'idx', 0)
limit = getarg(kwargs, 'limit', 50)
coll = kwargs.get('collection', 'merge')
view = kwargs.get('view', '')
if view == 'json':
res = []
inst = kwargs.get('instance', self.dbs_global)
form = self.form(uinput="")
check, content = self.generate_dasquery(query, inst)
if check:
return self.page(form + content, ctime=time.time()-time0)
dasquery = content # returned content is valid DAS query
nresults = self.dasmgr.rawcache.nresults(dasquery, coll)
gen = self.dasmgr.rawcache.get_from_cache\
(dasquery, idx=idx, limit=limit, collection=coll)
if recordid: # we got id
for row in gen:
if view == 'json':
res.append(row)
else:
res += das_json(dasquery, row)
else:
for row in gen:
rid = row['_id']
del row['_id']
res += self.templatepage('das_record', \
id=rid, collection=coll, daskeys=', '.join(row))
if recordid:
page = res
else:
url = '/das/records?'
if nresults:
page = self.templatepage('das_pagination', \
nrows=nresults, idx=idx, limit=limit, url=url, \
cgi=cgi, str=str)
else:
page = 'No results found, nresults=%s' % nresults
page += res
ctime = (time.time()-time0)
if view == 'json':
return json.dumps(res)
page = self.page(form + page, ctime=ctime)
return page
except Exception as exc:
print_exc(exc)
return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs):
"""
Retieve all records id's.
"""
try:
recordid = None
if args:
recordid = args[0]
spec = {"_id": ObjectId(recordid)}
fields = None
query = dict(fields=fields, spec=spec)
elif kwargs and "_id" in kwargs:
spec = {"_id": ObjectId(kwargs["_id"])}
fields = None
query = dict(fields=fields, spec=spec)
else: # return all ids
query = dict(fields=None, spec={})
res = ""
time0 = time.time()
idx = getarg(kwargs, "idx", 0)
limit = getarg(kwargs, "limit", 10)
coll = kwargs.get("collection", "merge")
view = kwargs.get("view", "")
if view == "json":
res = []
inst = kwargs.get("instance", self.dbs_global)
form = self.form(uinput="")
check, content = self.generate_dasquery(query, inst)
if check:
return self.page(form + content, ctime=time.time() - time0)
dasquery = content # returned content is valid DAS query
nresults = self.dasmgr.rawcache.nresults(dasquery, coll)
gen = self.dasmgr.rawcache.get_from_cache(dasquery, idx=idx, limit=limit, collection=coll)
if recordid: # we got id
for row in gen:
if view == "json":
res.append(row)
else:
res += das_json(dasquery, row)
else:
for row in gen:
rid = row["_id"]
del row["_id"]
res += self.templatepage("das_record", id=rid, collection=coll, daskeys=", ".join(row))
if recordid:
page = res
else:
url = "/das/records?"
if nresults:
page = self.templatepage("das_pagination", nrows=nresults, idx=idx, limit=limit, url=url)
else:
page = "No results found, nresults=%s" % nresults
page += res
ctime = time.time() - time0
if view == "json":
return json.dumps(res)
page = self.page(form + page, ctime=ctime)
return page
except Exception as exc:
print_exc(exc)
return self.error(gen_error_msg(kwargs))