def test_identical_data_records(self): """Tests identical_data_records function""" old = {'das_id':1, 'test':1, 'das':1} new = {'das_id':2, 'test':1, 'das':'foo'} result = identical_data_records(old, new) self.assertEqual(True, result) row = {'das_id':3, 'test':1, 'name':'foo'} result = identical_data_records(old, row) self.assertEqual(False, result)
def test_identical_data_records(self): """Tests identical_data_records function""" old = {'das_id': 1, 'test': 1, 'das': 1} new = {'das_id': 2, 'test': 1, 'das': 'foo'} result = identical_data_records(old, new) self.assertEqual(True, result) row = {'das_id': 3, 'test': 1, 'name': 'foo'} result = identical_data_records(old, row) self.assertEqual(False, result)
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) apilist = head.get('apilist') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters aggrtrs = dasquery.aggregators main = self.pagination(head) style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: page += '<br/><span class="box_red">%s</span>' % reason for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None pval = None lkey = None if 'das' in row and 'primary_key' in row['das']: pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(row) try: lkey = pkey.split('.')[0] if pkey == 'summary': pval = row[pkey] else: pval = [i for i in DotDict(row).get_values(pkey)] if isinstance(pval, list): if pval and not isinstance(pval[0], list): pval = list(set(pval)) else: pval = list(set(pval)) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': if isinstance(pval, basestring): pval = int(pval) except Exception as exc: msg = "Fail to extract pval for pkey='%s', lkey='%s'" \ % (pkey, lkey) msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval)) print msg print_exc(exc) pval = 'N/A' try: if not filters: if pkey == 'summary': page += 'Summary information:' elif pval and pval != 'N/A': page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: args = make_args(lkey, pval, inst) page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (args, pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if 'link' in item: linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = '<a href="/das/download?lfn=%s">Download</a>'\ % lfn if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except Exception as exc: print_exc(exc) pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation if filters and not aggrtrs: page += add_filter_values(row, filters) else: page += adjust_values(func, gen, links, pkey) pad = "" try: if 'das' in row and 'system' in row['das']: systems = self.systems(row['das']['system']) else: systems = "" # no das record print dastimestamp('DAS ERROR '), \ 'record without DAS key', row except KeyError as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(dasquery, row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if 'das' in row and 'conflict' in row['das']: conflict = ', '.join(row['das']['conflict']) else: conflict = '' page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict) page += '</div>' old = row main += fltpage if dup and not dasquery.aggregators: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page proc_time = self.processing_time(dasquery) if proc_time: msg = 'processing time: %5.3f sec, ' % proc_time else: msg = '' msg += 'cache server time: %5.3f sec' % head['ctime'] main += '<div align="right">%s</div>' % msg return main
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) incache = head.get('incache') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters main = self.pagination(total, incache, kwargs) if main.find('das_noresults') == -1: main += self.templatepage('das_colors', colors=self.colors) style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None lkey = None if row.has_key('das') and row['das'].has_key('primary_key'): pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(row) try: lkey = pkey.split('.')[0] pval = list(set(DotDict(row).get_values(pkey))) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': pval = int(pval) if pval: page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, pval, inst), pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if item.has_key('link'): linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = self.templatepage(\ 'filemover', lfn=lfn) if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except: pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation page += add_filter_values(row, filters) page += adjust_values(func, gen, links) pad = "" try: systems = self.systems(row['das']['system']) if row['das']['system'] == ['combined'] or \ row['das']['system'] == [u'combined']: if lkey: rowsystems = DotDict(row).get('%s.combined' % lkey) try: systems = self.systems(rowsystems) except TypeError as _err: systems = self.systems(['combined']) except KeyError: systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if row.has_key('das') and row['das'].has_key('conflict'): conflict = ', '.join(row['das']['conflict']) else: conflict = '' page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict) page += '</div>' old = row main += fltpage if dup: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page main += '<div align="right">DAS cache server time: %5.3f sec</div>' \ % head['ctime'] return main
def listview(self, head, data): """ Represent data in list view. """ kwargs = head.get('args') uinput = kwargs.get('input', '') total = head.get('nresults', 0) apilist = head.get('apilist') dasquery = head.get('dasquery', None) if not dasquery: inst = head.get('instance', self.dbs_global) dasquery = DASQuery(uinput, instance=inst) inst = dasquery.instance filters = dasquery.filters aggrtrs = dasquery.aggregators pager = self.pagination(head) main = pager style = 'white' rowkeys = [] fltpage = self.filter_bar(dasquery) page = '' old = None dup = False status = head.get('status', None) if status == 'fail': reason = head.get('reason', '') if reason: page += '<br/><span class="box_red">%s</span>' % reason for row in data: if not row: continue if not dup and old and identical_data_records(old, row): dup = True error = row.get('error', None) try: mongo_id = row['_id'] except Exception as exc: msg = 'Exception: %s\n' % str(exc) msg += 'Fail to process row\n%s' % str(row) raise Exception(msg) page += '<div class="%s"><hr class="line" />' % style links = [] pkey = None pval = None lkey = None if 'das' in row and 'primary_key' in row['das']: pkey = row['das']['primary_key'] if pkey and not rowkeys and not fltpage: fltpage = self.fltpage(dasquery) try: lkey = pkey.split('.')[0] if pkey == 'summary': pval = row[pkey] else: pval = [i for i in DotDict(row).get_values(pkey)] if isinstance(pval, list): if pval and not isinstance(pval[0], list): pval = list(set(pval)) else: pval = list(set(pval)) if len(pval) == 1: pval = pval[0] if pkey == 'run.run_number' or pkey == 'lumi.number': if isinstance(pval, basestring): pval = int(pval) except Exception as exc: msg = "Fail to extract pval for pkey='%s', lkey='%s'" \ % (pkey, lkey) msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval)) print(msg) print_exc(exc) pval = 'N/A' try: if not filters: if pkey == 'summary': page += 'Summary information:' elif pval and pval != 'N/A': page += '%s: ' % lkey.capitalize() if lkey == 'parent' or lkey == 'child': if str(pval).find('.root') != -1: lkey = 'file' else: lkey = 'dataset' if lkey in not_to_link(): page += '%s' % pval elif isinstance(pval, list): page += ', '.join(['<span class="highlight>"'+\ '<a href="/das/request?%s">%s</a></span>'\ % (make_args(lkey, i, inst), i) for i in pval]) else: args = make_args(lkey, pval, inst) page += '<span class="highlight">'+\ '<a href="/das/request?%s">%s</a></span>'\ % (args, pval) else: page += '%s: N/A' % lkey.capitalize() plist = self.dasmgr.mapping.presentation(lkey) linkrec = None for item in plist: if 'link' in item: linkrec = item['link'] break if linkrec and pval and pval != 'N/A' and \ not isinstance(pval, list) and not error: links += [l for l in make_links(linkrec, pval, inst)] if pkey and pkey == 'file.name': try: lfn = DotDict(row).get('file.name') val = '<a href="/das/download?lfn=%s">Download</a>'\ % lfn if lfn else '' if val: links.append(val) except: pass if pkey and pkey == 'site.name': try: site = DotDict(row).get('site.name') val = self.templatepage(\ 'sitedb', item=site, api="sites") if site else '' if val: links.append(val) except: pass if pkey and pkey == 'user.name': try: user = DotDict(row).get('user.username') val = self.templatepage(\ 'sitedb', item=user, api="people") if user else '' if val: links.append(val) except: pass if pkey and pkey == 'dataset.name': try: path = DotDict(row).get('dataset.name') if path: links.append(self.templatepage(\ 'makepy', path=path, inst=inst)) if inst == self.dbs_global: links.append(self.templatepage(\ 'phedex_subscription', path=path)) links.append(self.templatepage(\ 'xsecdb', primds=path.split('/')[1])) except: pass if pkey and pkey == 'release.name': rel = '["%s"]' % DotDict(row).get('release.name') url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?' url += 'diff=false&releases=%s' % urllib.quote(rel) links.append('<a href="%s">Packages</a>' % url) except Exception as exc: print_exc(exc) pval = 'N/A' gen = self.convert2ui(row, pkey) if self.dasmgr: func = self.dasmgr.mapping.daskey_from_presentation if filters and not aggrtrs: page += add_filter_values(row, filters) else: page += adjust_values(func, gen, links, pkey) pad = "" try: if 'das' in row and 'system' in row['das']: systems = self.systems(row['das']['system']) else: systems = "" # no das record print(dastimestamp('DAS ERROR '), \ 'record without DAS key', row) except KeyError as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records except Exception as exc: print_exc(exc) systems = "" # we don't store systems for aggregated records jsonhtml = das_json(dasquery, row, pad) jsonhtml = jsonhtml.replace(\ 'request?', 'request?instance=%s&' % inst) if not links: page += '<br />' if 'das' in row and 'conflict' in row['das']: conflict = ', '.join(row['das']['conflict']) else: conflict = '' hints = '' for hint in row.get('hints', {}): if hint: hints += self.templatepage('hint', hint=hint, base=self.base, dbs=self.dbs_global) page += self.templatepage('das_row', systems=systems, \ sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id, conflict=conflict, hints=hints) page += '</div>' old = row main += fltpage if dup and not dasquery.aggregators: main += self.templatepage('das_duplicates', uinput=uinput, instance=inst) main += page if total>10: main += '<hr class="line" />' main += pager main += '<hr class="line" />' proc_time = self.processing_time(dasquery) if proc_time: msg = 'processing time: %5.3f sec, ' % proc_time else: msg = '' msg += 'cache server time: %5.3f sec' % head['ctime'] main += '<div align="right">%s</div>' % msg return main