def api(self, name, **kwargs): """ Return DAS mapping record about provided API. """ record = self.dasmgr.mapping.api_info(name) page = "<b>DAS mapping record</b>" page += das_json(record) return self.page(page, response_div=False)
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None if args: recordid = args[0] spec = {'_id':ObjectId(recordid)} fields = None query = dict(fields=fields, spec=spec) elif kwargs and kwargs.has_key('_id'): spec = {'_id': ObjectId(kwargs['_id'])} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) res = '' time0 = time.time() idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) coll = kwargs.get('collection', 'merge') inst = kwargs.get('instance', self.dbs_global) form = self.form(uinput="") check, content = self.generate_dasquery(query, inst) if check: return self.page(form + content, ctime=time.time()-time0) dasquery = content # returned content is valid DAS query nresults = self.dasmgr.rawcache.nresults(dasquery, coll) gen = self.dasmgr.rawcache.get_from_cache\ (dasquery, idx=idx, limit=limit, collection=coll) if recordid: # we got id for row in gen: res += das_json(row) else: for row in gen: rid = row['_id'] del row['_id'] res += self.templatepage('das_record', \ id=rid, collection=coll, daskeys=', '.join(row)) if recordid: page = res else: url = '/das/records?' if nresults: page = self.templatepage('das_pagination', \ nrows=nresults, idx=idx, limit=limit, url=url) else: page = 'No results found, nresults=%s' % nresults page += res ctime = (time.time()-time0) page = self.page(form + page, ctime=ctime) return page except Exception as exc: print_exc(exc) return self.error(gen_error_msg(kwargs))
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') total = head.get('nresults', 0) incache = head.get('incache') main = self.pagination(total, incache, kwargs) style = 'white' page = '' pad = '' for row in data: if not row: continue if style == 'white': style = 'gray' else: style = 'white' try: mongo_id = row['_id'] except Exception as exc: msg = str(exc) msg += '\nFail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style jsonhtml = das_json(row, pad) if row.has_key('das') and row['das'].has_key('conflict'): conflict = ', '.join(row['das']['conflict']) else: conflict = '' page += self.templatepage('das_row', systems='', \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict) page += '</div>' main += page main += '<div align="right">DAS cache server time: %5.3f sec</div>' \ % head['ctime'] return main
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) apilist = head.get('apilist') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters aggrtrs = dasquery.aggregators main = self.pagination(head) style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: page += '<br/><span class="box_red">%s</span>' % reason for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None pval = None lkey = None if 'das' in row and 'primary_key' in row['das']: pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(row) try: lkey = pkey.split('.')[0] if pkey == 'summary': pval = row[pkey] else: pval = [i for i in DotDict(row).get_values(pkey)] if isinstance(pval, list): if pval and not isinstance(pval[0], list): pval = list(set(pval)) else: pval = list(set(pval)) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': if isinstance(pval, basestring): pval = int(pval) except Exception as exc: msg = "Fail to extract pval for pkey='%s', lkey='%s'" \ % (pkey, lkey) msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval)) print msg print_exc(exc) pval = 'N/A' try: if not filters: if pkey == 'summary': page += 'Summary information:' elif pval and pval != 'N/A': page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: args = make_args(lkey, pval, inst) page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (args, pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if 'link' in item: linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = '<a href="/das/download?lfn=%s">Download</a>'\ % lfn if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except Exception as exc: print_exc(exc) pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation if filters and not aggrtrs: page += add_filter_values(row, filters) else: page += adjust_values(func, gen, links, pkey) pad = "" try: if 'das' in row and 'system' in row['das']: systems = self.systems(row['das']['system']) else: systems = "" # no das record print dastimestamp('DAS ERROR '), \ 'record without DAS key', row except KeyError as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(dasquery, row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if 'das' in row and 'conflict' in row['das']: conflict = ', '.join(row['das']['conflict']) else: conflict = '' page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict) page += '</div>' old = row main += fltpage if dup and not dasquery.aggregators: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page proc_time = self.processing_time(dasquery) if proc_time: msg = 'processing time: %5.3f sec, ' % proc_time else: msg = '' msg += 'cache server time: %5.3f sec' % head['ctime'] main += '<div align="right">%s</div>' % msg return main
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) incache = head.get('incache') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters main = self.pagination(total, incache, kwargs) if main.find('das_noresults') == -1: main += self.templatepage('das_colors', colors=self.colors) style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None lkey = None if row.has_key('das') and row['das'].has_key('primary_key'): pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(row) try: lkey = pkey.split('.')[0] pval = list(set(DotDict(row).get_values(pkey))) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': pval = int(pval) if pval: page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, pval, inst), pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if item.has_key('link'): linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = self.templatepage(\ 'filemover', lfn=lfn) if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except: pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation page += add_filter_values(row, filters) page += adjust_values(func, gen, links) pad = "" try: systems = self.systems(row['das']['system']) if row['das']['system'] == ['combined'] or \ row['das']['system'] == [u'combined']: if lkey: rowsystems = DotDict(row).get('%s.combined' % lkey) try: systems = self.systems(rowsystems) except TypeError as _err: systems = self.systems(['combined']) except KeyError: systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if row.has_key('das') and row['das'].has_key('conflict'): conflict = ', '.join(row['das']['conflict']) else: conflict = '' page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict) page += '</div>' old = row main += fltpage if dup: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page main += '<div align="right">DAS cache server time: %5.3f sec</div>' \ % head['ctime'] return main
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) apilist = head.get('apilist') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters aggrtrs = dasquery.aggregators pager = self.pagination(head) main = pager style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: page += '<br/><span class="box_red">%s</span>' % reason for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None pval = None lkey = None if 'das' in row and 'primary_key' in row['das']: pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(dasquery) try: lkey = pkey.split('.')[0] if pkey == 'summary': pval = row[pkey] else: pval = [i for i in DotDict(row).get_values(pkey)] if isinstance(pval, list): if pval and not isinstance(pval[0], list): pval = list(set(pval)) else: pval = list(set(pval)) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': if isinstance(pval, basestring): pval = int(pval) except Exception as exc: msg = "Fail to extract pval for pkey='%s', lkey='%s'" \ % (pkey, lkey) msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval)) print(msg) print_exc(exc) pval = 'N/A' try: if not filters: if pkey == 'summary': page += 'Summary information:' elif pval and pval != 'N/A': page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: args = make_args(lkey, pval, inst) page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (args, pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if 'link' in item: linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = '<a href="/das/download?lfn=%s">Download</a>'\ % lfn if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) links.append(self.templatepage(\ 'xsecdb', primds=path.split('/')[1])) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except Exception as exc: print_exc(exc) pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation if filters and not aggrtrs: page += add_filter_values(row, filters) else: page += adjust_values(func, gen, links, pkey) pad = "" try: if 'das' in row and 'system' in row['das']: systems = self.systems(row['das']['system']) else: systems = "" # no das record print(dastimestamp('DAS ERROR '), \ 'record without DAS key', row) except KeyError as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(dasquery, row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if 'das' in row and 'conflict' in row['das']: conflict = ', '.join(row['das']['conflict']) else: conflict = '' hints = '' for hint in row.get('hints', {}): if hint: hints += self.templatepage('hint', hint=hint, base=self.base, dbs=self.dbs_global) page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict, hints=hints) page += '</div>' old = row main += fltpage if dup and not dasquery.aggregators: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page if total>10: main += '<hr class="line" />' main += pager main += '<hr class="line" />' proc_time = self.processing_time(dasquery) if proc_time: msg = 'processing time: %5.3f sec, ' % proc_time else: msg = '' msg += 'cache server time: %5.3f sec' % head['ctime'] main += '<div align="right">%s</div>' % msg return main
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None if args: recordid = args[0] spec = {'_id':ObjectId(recordid)} fields = None query = dict(fields=fields, spec=spec) elif kwargs and '_id' in kwargs: spec = {'_id': ObjectId(kwargs['_id'])} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) res = '' time0 = time.time() idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 50) coll = kwargs.get('collection', 'merge') view = kwargs.get('view', '') if view == 'json': res = [] inst = kwargs.get('instance', self.dbs_global) form = self.form(uinput="") check, content = self.generate_dasquery(query, inst) if check: return self.page(form + content, ctime=time.time()-time0) dasquery = content # returned content is valid DAS query nresults = self.dasmgr.rawcache.nresults(dasquery, coll) gen = self.dasmgr.rawcache.get_from_cache\ (dasquery, idx=idx, limit=limit, collection=coll) if recordid: # we got id for row in gen: if view == 'json': res.append(row) else: res += das_json(dasquery, row) else: for row in gen: rid = row['_id'] del row['_id'] res += self.templatepage('das_record', \ id=rid, collection=coll, daskeys=', '.join(row)) if recordid: page = res else: url = '/das/records?' if nresults: page = self.templatepage('das_pagination', \ nrows=nresults, idx=idx, limit=limit, url=url, \ cgi=cgi, str=str) else: page = 'No results found, nresults=%s' % nresults page += res ctime = (time.time()-time0) if view == 'json': return json.dumps(res) page = self.page(form + page, ctime=ctime) return page except Exception as exc: print_exc(exc) return self.error(gen_error_msg(kwargs))
def records(self, *args, **kwargs): """ Retieve all records id's. """ try: recordid = None if args: recordid = args[0] spec = {"_id": ObjectId(recordid)} fields = None query = dict(fields=fields, spec=spec) elif kwargs and "_id" in kwargs: spec = {"_id": ObjectId(kwargs["_id"])} fields = None query = dict(fields=fields, spec=spec) else: # return all ids query = dict(fields=None, spec={}) res = "" time0 = time.time() idx = getarg(kwargs, "idx", 0) limit = getarg(kwargs, "limit", 10) coll = kwargs.get("collection", "merge") view = kwargs.get("view", "") if view == "json": res = [] inst = kwargs.get("instance", self.dbs_global) form = self.form(uinput="") check, content = self.generate_dasquery(query, inst) if check: return self.page(form + content, ctime=time.time() - time0) dasquery = content # returned content is valid DAS query nresults = self.dasmgr.rawcache.nresults(dasquery, coll) gen = self.dasmgr.rawcache.get_from_cache(dasquery, idx=idx, limit=limit, collection=coll) if recordid: # we got id for row in gen: if view == "json": res.append(row) else: res += das_json(dasquery, row) else: for row in gen: rid = row["_id"] del row["_id"] res += self.templatepage("das_record", id=rid, collection=coll, daskeys=", ".join(row)) if recordid: page = res else: url = "/das/records?" if nresults: page = self.templatepage("das_pagination", nrows=nresults, idx=idx, limit=limit, url=url) else: page = "No results found, nresults=%s" % nresults page += res ctime = time.time() - time0 if view == "json": return json.dumps(res) page = self.page(form + page, ctime=ctime) return page except Exception as exc: print_exc(exc) return self.error(gen_error_msg(kwargs))