Ejemplo n.º 1
0
Archivo: utils_t.py Proyecto: ktf/DAS
 def test_identical_data_records(self):
     """Tests identical_data_records function"""
     old = {'das_id':1, 'test':1, 'das':1}
     new = {'das_id':2, 'test':1, 'das':'foo'}
     result = identical_data_records(old, new)
     self.assertEqual(True, result)
     row = {'das_id':3, 'test':1, 'name':'foo'}
     result = identical_data_records(old, row)
     self.assertEqual(False, result)
Ejemplo n.º 2
0
 def test_identical_data_records(self):
     """Tests identical_data_records function"""
     old = {'das_id': 1, 'test': 1, 'das': 1}
     new = {'das_id': 2, 'test': 1, 'das': 'foo'}
     result = identical_data_records(old, new)
     self.assertEqual(True, result)
     row = {'das_id': 3, 'test': 1, 'name': 'foo'}
     result = identical_data_records(old, row)
     self.assertEqual(False, result)
Ejemplo n.º 3
0
 def listview(self, head, data):
     """
     Represent data in list view.
     """
     kwargs   = head.get('args')
     uinput   = kwargs.get('input', '')
     total    = head.get('nresults', 0)
     apilist  = head.get('apilist')
     dasquery = head.get('dasquery', None)
     if  not dasquery:
         inst     = head.get('instance', self.dbs_global)
         dasquery = DASQuery(uinput, instance=inst)
     inst     = dasquery.instance
     filters  = dasquery.filters
     aggrtrs  = dasquery.aggregators
     main     = self.pagination(head)
     style    = 'white'
     rowkeys  = []
     fltpage  = self.filter_bar(dasquery)
     page     = ''
     old      = None
     dup      = False
     status   = head.get('status', None)
     if  status == 'fail':
         reason = head.get('reason', '')
         if  reason:
             page += '<br/><span class="box_red">%s</span>' % reason
     for row in data:
         if  not row:
             continue
         if  not dup and old and identical_data_records(old, row):
             dup = True
         error = row.get('error', None)
         try:
             mongo_id = row['_id']
         except Exception as exc:
             msg  = 'Exception: %s\n' % str(exc)
             msg += 'Fail to process row\n%s' % str(row)
             raise Exception(msg)
         page += '<div class="%s"><hr class="line" />' % style
         links = []
         pkey  = None
         pval  = None
         lkey  = None
         if  'das' in row and 'primary_key' in row['das']:
             pkey = row['das']['primary_key']
             if  pkey and not rowkeys and not fltpage:
                 fltpage = self.fltpage(row)
             try:
                 lkey = pkey.split('.')[0]
                 if  pkey == 'summary':
                     pval = row[pkey]
                 else:
                     pval = [i for i in DotDict(row).get_values(pkey)]
                     if  isinstance(pval, list):
                         if  pval and not isinstance(pval[0], list):
                             pval = list(set(pval))
                     else:
                         pval = list(set(pval))
                     if  len(pval) == 1:
                         pval = pval[0]
                     if  pkey == 'run.run_number' or pkey == 'lumi.number':
                         if  isinstance(pval, basestring):
                             pval = int(pval)
             except Exception as exc:
                 msg  = "Fail to extract pval for pkey='%s', lkey='%s'" \
                         % (pkey, lkey)
                 msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
                 print msg
                 print_exc(exc)
                 pval = 'N/A'
             try:
                 if  not filters:
                     if  pkey == 'summary':
                         page += 'Summary information:'
                     elif  pval and pval != 'N/A':
                         page += '%s: ' % lkey.capitalize()
                         if  lkey == 'parent' or lkey == 'child':
                             if  str(pval).find('.root') != -1:
                                 lkey = 'file'
                             else:
                                 lkey = 'dataset'
                         if  lkey in not_to_link():
                             page += '%s' % pval
                         elif  isinstance(pval, list):
                             page += ', '.join(['<span class="highlight>"'+\
                                 '<a href="/das/request?%s">%s</a></span>'\
                                 % (make_args(lkey, i, inst), i) for i in pval])
                         else:
                             args  = make_args(lkey, pval, inst)
                             page += '<span class="highlight">'+\
                                 '<a href="/das/request?%s">%s</a></span>'\
                                 % (args, pval)
                     else:
                         page += '%s: N/A' % lkey.capitalize()
                 plist = self.dasmgr.mapping.presentation(lkey)
                 linkrec = None
                 for item in plist:
                     if  'link' in item:
                         linkrec = item['link']
                         break
                 if  linkrec and pval and pval != 'N/A' and \
                     not isinstance(pval, list) and not error:
                     links += [l for l in make_links(linkrec, pval, inst)]
                 if  pkey and pkey == 'file.name':
                     try:
                         lfn = DotDict(row).get('file.name')
                         val = '<a href="/das/download?lfn=%s">Download</a>'\
                                 % lfn if lfn else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'site.name':
                     try:
                         site = DotDict(row).get('site.name')
                         val = self.templatepage(\
                         'sitedb', item=site, api="sites") if site else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'user.name':
                     try:
                         user = DotDict(row).get('user.username')
                         val = self.templatepage(\
                         'sitedb', item=user, api="people") if user else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'dataset.name':
                     try:
                         path = DotDict(row).get('dataset.name')
                         if  path:
                             links.append(self.templatepage(\
                                 'makepy', path=path, inst=inst))
                             if  inst == self.dbs_global:
                                 links.append(self.templatepage(\
                                     'phedex_subscription', path=path))
                     except:
                         pass
                 if  pkey and pkey == 'release.name':
                     rel  = '["%s"]' % DotDict(row).get('release.name')
                     url  = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
                     url += 'diff=false&releases=%s' % urllib.quote(rel)
                     links.append('<a href="%s">Packages</a>' % url)
             except Exception as exc:
                 print_exc(exc)
                 pval = 'N/A'
         gen   = self.convert2ui(row, pkey)
         if  self.dasmgr:
             func  = self.dasmgr.mapping.daskey_from_presentation
             if  filters and not aggrtrs:
                 page += add_filter_values(row, filters)
             else:
                 page += adjust_values(func, gen, links, pkey)
         pad   = ""
         try:
             if  'das' in row and 'system' in row['das']:
                 systems = self.systems(row['das']['system'])
             else:
                 systems = "" # no das record
                 print dastimestamp('DAS ERROR '), \
                         'record without DAS key', row
         except KeyError as exc:
             print_exc(exc)
             systems = "" # we don't store systems for aggregated records
         except Exception as exc:
             print_exc(exc)
             systems = "" # we don't store systems for aggregated records
         jsonhtml = das_json(dasquery, row, pad)
         jsonhtml = jsonhtml.replace(\
             'request?', 'request?instance=%s&' % inst)
         if  not links:
             page += '<br />'
         if  'das' in row and 'conflict' in row['das']:
             conflict = ', '.join(row['das']['conflict'])
         else:
             conflict = ''
         page += self.templatepage('das_row', systems=systems, \
                 sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
                 conflict=conflict)
         page += '</div>'
         old = row
     main += fltpage
     if  dup and not dasquery.aggregators:
         main += self.templatepage('das_duplicates', uinput=uinput,
                     instance=inst)
     main += page
     proc_time = self.processing_time(dasquery)
     if  proc_time:
         msg = 'processing time: %5.3f sec, ' % proc_time
     else:
         msg   = ''
     msg  += 'cache server time: %5.3f sec' % head['ctime']
     main += '<div align="right">%s</div>' % msg
     return main
Ejemplo n.º 4
0
 def listview(self, head, data):
     """
     Represent data in list view.
     """
     kwargs   = head.get('args')
     uinput   = kwargs.get('input', '')
     total    = head.get('nresults', 0)
     incache  = head.get('incache')
     dasquery = head.get('dasquery', None)
     if  not dasquery:
         inst     = head.get('instance', self.dbs_global)
         dasquery = DASQuery(uinput, instance=inst)
     inst     = dasquery.instance
     filters  = dasquery.filters
     main     = self.pagination(total, incache, kwargs)
     if  main.find('das_noresults') == -1:
         main += self.templatepage('das_colors', colors=self.colors)
     style   = 'white'
     rowkeys = []
     fltpage = self.filter_bar(dasquery)
     page    = ''
     old     = None
     dup     = False
     for row in data:
         if  not row:
             continue
         if  not dup and old and identical_data_records(old, row):
             dup = True
         error = row.get('error', None)
         try:
             mongo_id = row['_id']
         except Exception as exc:
             msg  = 'Exception: %s\n' % str(exc)
             msg += 'Fail to process row\n%s' % str(row)
             raise Exception(msg)
         page += '<div class="%s"><hr class="line" />' % style
         links = []
         pkey  = None
         lkey  = None
         if  row.has_key('das') and row['das'].has_key('primary_key'):
             pkey = row['das']['primary_key']
             if  pkey and not rowkeys and not fltpage:
                 fltpage = self.fltpage(row)
             try:
                 lkey = pkey.split('.')[0]
                 pval = list(set(DotDict(row).get_values(pkey)))
                 if  len(pval) == 1:
                     pval = pval[0]
                 if  pkey == 'run.run_number' or pkey == 'lumi.number':
                     pval = int(pval)
                 if  pval:
                     page += '%s: ' % lkey.capitalize()
                     if  lkey == 'parent' or lkey == 'child':
                         if  str(pval).find('.root') != -1:
                             lkey = 'file'
                         else:
                             lkey = 'dataset'
                     if  lkey in not_to_link():
                         page += '%s' % pval
                     elif  isinstance(pval, list):
                         page += ', '.join(['<span class="highlight>"'+\
                             '<a href="/das/request?%s">%s</a></span>'\
                             % (make_args(lkey, i, inst), i) for i in pval])
                     else:
                         page += '<span class="highlight">'+\
                             '<a href="/das/request?%s">%s</a></span>'\
                             % (make_args(lkey, pval, inst), pval)
                 else:
                     page += '%s: N/A' % lkey.capitalize()
                 plist = self.dasmgr.mapping.presentation(lkey)
                 linkrec = None
                 for item in plist:
                     if  item.has_key('link'):
                         linkrec = item['link']
                         break
                 if  linkrec and pval and pval != 'N/A' and \
                     not isinstance(pval, list) and not error:
                     links += [l for l in make_links(linkrec, pval, inst)]
                 if  pkey and pkey == 'file.name':
                     try:
                         lfn = DotDict(row).get('file.name')
                         val = self.templatepage(\
                             'filemover', lfn=lfn) if lfn else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'site.name':
                     try:
                         site = DotDict(row).get('site.name')
                         val = self.templatepage(\
                         'sitedb', item=site, api="sites") if site else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'user.name':
                     try:
                         user = DotDict(row).get('user.username')
                         val = self.templatepage(\
                         'sitedb', item=user, api="people") if user else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'dataset.name':
                     try:
                         path = DotDict(row).get('dataset.name')
                         if  path:
                             links.append(self.templatepage(\
                                 'makepy', path=path, inst=inst))
                             if  inst == self.dbs_global:
                                 links.append(self.templatepage(\
                                     'phedex_subscription', path=path))
                     except:
                         pass
                 if  pkey and pkey == 'release.name':
                     rel  = '["%s"]' % DotDict(row).get('release.name')
                     url  = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
                     url += 'diff=false&releases=%s' % urllib.quote(rel)
                     links.append('<a href="%s">Packages</a>' % url)
             except:
                 pval = 'N/A'
         gen   = self.convert2ui(row, pkey)
         if  self.dasmgr:
             func  = self.dasmgr.mapping.daskey_from_presentation
             page += add_filter_values(row, filters)
             page += adjust_values(func, gen, links)
         pad   = ""
         try:
             systems = self.systems(row['das']['system'])
             if  row['das']['system'] == ['combined'] or \
                 row['das']['system'] == [u'combined']:
                 if  lkey:
                     rowsystems = DotDict(row).get('%s.combined' % lkey) 
                     try:
                         systems = self.systems(rowsystems)
                     except TypeError as _err:
                         systems = self.systems(['combined'])
         except KeyError:
             systems = "" # we don't store systems for aggregated records
         except Exception as exc:
             print_exc(exc)
             systems = "" # we don't store systems for aggregated records
         jsonhtml = das_json(row, pad)
         jsonhtml = jsonhtml.replace(\
             'request?', 'request?instance=%s&' % inst)
         if  not links:
             page += '<br />'
         if  row.has_key('das') and row['das'].has_key('conflict'):
             conflict = ', '.join(row['das']['conflict'])
         else:
             conflict = ''
         page += self.templatepage('das_row', systems=systems, \
                 sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
                 conflict=conflict)
         page += '</div>'
         old = row
     main += fltpage
     if  dup:
         main += self.templatepage('das_duplicates', uinput=uinput,
                     instance=inst)
     main += page
     main += '<div align="right">DAS cache server time: %5.3f sec</div>' \
             % head['ctime']
     return main
Ejemplo n.º 5
0
 def listview(self, head, data):
     """
     Represent data in list view.
     """
     kwargs   = head.get('args')
     uinput   = kwargs.get('input', '')
     total    = head.get('nresults', 0)
     apilist  = head.get('apilist')
     dasquery = head.get('dasquery', None)
     if  not dasquery:
         inst     = head.get('instance', self.dbs_global)
         dasquery = DASQuery(uinput, instance=inst)
     inst     = dasquery.instance
     filters  = dasquery.filters
     aggrtrs  = dasquery.aggregators
     pager    = self.pagination(head)
     main     = pager
     style    = 'white'
     rowkeys  = []
     fltpage  = self.filter_bar(dasquery)
     page     = ''
     old      = None
     dup      = False
     status   = head.get('status', None)
     if  status == 'fail':
         reason = head.get('reason', '')
         if  reason:
             page += '<br/><span class="box_red">%s</span>' % reason
     for row in data:
         if  not row:
             continue
         if  not dup and old and identical_data_records(old, row):
             dup = True
         error = row.get('error', None)
         try:
             mongo_id = row['_id']
         except Exception as exc:
             msg  = 'Exception: %s\n' % str(exc)
             msg += 'Fail to process row\n%s' % str(row)
             raise Exception(msg)
         page += '<div class="%s"><hr class="line" />' % style
         links = []
         pkey  = None
         pval  = None
         lkey  = None
         if  'das' in row and 'primary_key' in row['das']:
             pkey = row['das']['primary_key']
             if  pkey and not rowkeys and not fltpage:
                 fltpage = self.fltpage(dasquery)
             try:
                 lkey = pkey.split('.')[0]
                 if  pkey == 'summary':
                     pval = row[pkey]
                 else:
                     pval = [i for i in DotDict(row).get_values(pkey)]
                     if  isinstance(pval, list):
                         if  pval and not isinstance(pval[0], list):
                             pval = list(set(pval))
                     else:
                         pval = list(set(pval))
                     if  len(pval) == 1:
                         pval = pval[0]
                     if  pkey == 'run.run_number' or pkey == 'lumi.number':
                         if  isinstance(pval, basestring):
                             pval = int(pval)
             except Exception as exc:
                 msg  = "Fail to extract pval for pkey='%s', lkey='%s'" \
                         % (pkey, lkey)
                 msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
                 print(msg)
                 print_exc(exc)
                 pval = 'N/A'
             try:
                 if  not filters:
                     if  pkey == 'summary':
                         page += 'Summary information:'
                     elif  pval and pval != 'N/A':
                         page += '%s: ' % lkey.capitalize()
                         if  lkey == 'parent' or lkey == 'child':
                             if  str(pval).find('.root') != -1:
                                 lkey = 'file'
                             else:
                                 lkey = 'dataset'
                         if  lkey in not_to_link():
                             page += '%s' % pval
                         elif  isinstance(pval, list):
                             page += ', '.join(['<span class="highlight>"'+\
                                 '<a href="/das/request?%s">%s</a></span>'\
                                 % (make_args(lkey, i, inst), i) for i in pval])
                         else:
                             args  = make_args(lkey, pval, inst)
                             page += '<span class="highlight">'+\
                                 '<a href="/das/request?%s">%s</a></span>'\
                                 % (args, pval)
                     else:
                         page += '%s: N/A' % lkey.capitalize()
                 plist = self.dasmgr.mapping.presentation(lkey)
                 linkrec = None
                 for item in plist:
                     if  'link' in item:
                         linkrec = item['link']
                         break
                 if  linkrec and pval and pval != 'N/A' and \
                     not isinstance(pval, list) and not error:
                     links += [l for l in make_links(linkrec, pval, inst)]
                 if  pkey and pkey == 'file.name':
                     try:
                         lfn = DotDict(row).get('file.name')
                         val = '<a href="/das/download?lfn=%s">Download</a>'\
                                 % lfn if lfn else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'site.name':
                     try:
                         site = DotDict(row).get('site.name')
                         val = self.templatepage(\
                         'sitedb', item=site, api="sites") if site else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'user.name':
                     try:
                         user = DotDict(row).get('user.username')
                         val = self.templatepage(\
                         'sitedb', item=user, api="people") if user else ''
                         if  val: links.append(val)
                     except:
                         pass
                 if  pkey and pkey == 'dataset.name':
                     try:
                         path = DotDict(row).get('dataset.name')
                         if  path:
                             links.append(self.templatepage(\
                                 'makepy', path=path, inst=inst))
                             if  inst == self.dbs_global:
                                 links.append(self.templatepage(\
                                     'phedex_subscription', path=path))
                                 links.append(self.templatepage(\
                                     'xsecdb', primds=path.split('/')[1]))
                     except:
                         pass
                 if  pkey and pkey == 'release.name':
                     rel  = '["%s"]' % DotDict(row).get('release.name')
                     url  = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
                     url += 'diff=false&releases=%s' % urllib.quote(rel)
                     links.append('<a href="%s">Packages</a>' % url)
             except Exception as exc:
                 print_exc(exc)
                 pval = 'N/A'
         gen   = self.convert2ui(row, pkey)
         if  self.dasmgr:
             func  = self.dasmgr.mapping.daskey_from_presentation
             if  filters and not aggrtrs:
                 page += add_filter_values(row, filters)
             else:
                 page += adjust_values(func, gen, links, pkey)
         pad   = ""
         try:
             if  'das' in row and 'system' in row['das']:
                 systems = self.systems(row['das']['system'])
             else:
                 systems = "" # no das record
                 print(dastimestamp('DAS ERROR '), \
                         'record without DAS key', row)
         except KeyError as exc:
             print_exc(exc)
             systems = "" # we don't store systems for aggregated records
         except Exception as exc:
             print_exc(exc)
             systems = "" # we don't store systems for aggregated records
         jsonhtml = das_json(dasquery, row, pad)
         jsonhtml = jsonhtml.replace(\
             'request?', 'request?instance=%s&' % inst)
         if  not links:
             page += '<br />'
         if  'das' in row and 'conflict' in row['das']:
             conflict = ', '.join(row['das']['conflict'])
         else:
             conflict = ''
         hints = ''
         for hint in row.get('hints', {}):
             if  hint:
                 hints += self.templatepage('hint',
                         hint=hint, base=self.base, dbs=self.dbs_global)
         page += self.templatepage('das_row', systems=systems, \
                 sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
                 conflict=conflict, hints=hints)
         page += '</div>'
         old = row
     main += fltpage
     if  dup and not dasquery.aggregators:
         main += self.templatepage('das_duplicates', uinput=uinput,
                     instance=inst)
     main += page
     if total>10:
         main += '<hr class="line" />'
         main += pager
         main += '<hr class="line" />'
     proc_time = self.processing_time(dasquery)
     if  proc_time:
         msg = 'processing time: %5.3f sec, ' % proc_time
     else:
         msg   = ''
     msg  += 'cache server time: %5.3f sec' % head['ctime']
     main += '<div align="right">%s</div>' % msg
     return main