def test_identical_data_records(self):
"""Tests identical_data_records function"""
old = {'das_id':1, 'test':1, 'das':1}
new = {'das_id':2, 'test':1, 'das':'foo'}
result = identical_data_records(old, new)
self.assertEqual(True, result)
row = {'das_id':3, 'test':1, 'name':'foo'}
result = identical_data_records(old, row)
self.assertEqual(False, result)
def test_identical_data_records(self):
"""Tests identical_data_records function"""
old = {'das_id': 1, 'test': 1, 'das': 1}
new = {'das_id': 2, 'test': 1, 'das': 'foo'}
result = identical_data_records(old, new)
self.assertEqual(True, result)
row = {'das_id': 3, 'test': 1, 'name': 'foo'}
result = identical_data_records(old, row)
self.assertEqual(False, result)
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
aggrtrs = dasquery.aggregators
main = self.pagination(head)
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
pval = None
lkey = None
if 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(row)
try:
lkey = pkey.split('.')[0]
if pkey == 'summary':
pval = row[pkey]
else:
pval = [i for i in DotDict(row).get_values(pkey)]
if isinstance(pval, list):
if pval and not isinstance(pval[0], list):
pval = list(set(pval))
else:
pval = list(set(pval))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
if isinstance(pval, basestring):
pval = int(pval)
except Exception as exc:
msg = "Fail to extract pval for pkey='%s', lkey='%s'" \
% (pkey, lkey)
msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
print msg
print_exc(exc)
pval = 'N/A'
try:
if not filters:
if pkey == 'summary':
page += 'Summary information:'
elif pval and pval != 'N/A':
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
args = make_args(lkey, pval, inst)
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (args, pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if 'link' in item:
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = '<a href="/das/download?lfn=%s">Download</a>'\
% lfn if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except Exception as exc:
print_exc(exc)
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
if filters and not aggrtrs:
page += add_filter_values(row, filters)
else:
page += adjust_values(func, gen, links, pkey)
pad = ""
try:
if 'das' in row and 'system' in row['das']:
systems = self.systems(row['das']['system'])
else:
systems = "" # no das record
print dastimestamp('DAS ERROR '), \
'record without DAS key', row
except KeyError as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(dasquery, row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if 'das' in row and 'conflict' in row['das']:
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict)
page += '</div>'
old = row
main += fltpage
if dup and not dasquery.aggregators:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
proc_time = self.processing_time(dasquery)
if proc_time:
msg = 'processing time: %5.3f sec, ' % proc_time
else:
msg = ''
msg += 'cache server time: %5.3f sec' % head['ctime']
main += '<div align="right">%s</div>' % msg
return main
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
incache = head.get('incache')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
main = self.pagination(total, incache, kwargs)
if main.find('das_noresults') == -1:
main += self.templatepage('das_colors', colors=self.colors)
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
lkey = None
if row.has_key('das') and row['das'].has_key('primary_key'):
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(row)
try:
lkey = pkey.split('.')[0]
pval = list(set(DotDict(row).get_values(pkey)))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
pval = int(pval)
if pval:
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, pval, inst), pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if item.has_key('link'):
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = self.templatepage(\
'filemover', lfn=lfn) if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except:
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
page += add_filter_values(row, filters)
page += adjust_values(func, gen, links)
pad = ""
try:
systems = self.systems(row['das']['system'])
if row['das']['system'] == ['combined'] or \
row['das']['system'] == [u'combined']:
if lkey:
rowsystems = DotDict(row).get('%s.combined' % lkey)
try:
systems = self.systems(rowsystems)
except TypeError as _err:
systems = self.systems(['combined'])
except KeyError:
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if row.has_key('das') and row['das'].has_key('conflict'):
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict)
page += '</div>'
old = row
main += fltpage
if dup:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
main += '<div align="right">DAS cache server time: %5.3f sec</div>' \
% head['ctime']
return main
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
aggrtrs = dasquery.aggregators
pager = self.pagination(head)
main = pager
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
pval = None
lkey = None
if 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(dasquery)
try:
lkey = pkey.split('.')[0]
if pkey == 'summary':
pval = row[pkey]
else:
pval = [i for i in DotDict(row).get_values(pkey)]
if isinstance(pval, list):
if pval and not isinstance(pval[0], list):
pval = list(set(pval))
else:
pval = list(set(pval))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
if isinstance(pval, basestring):
pval = int(pval)
except Exception as exc:
msg = "Fail to extract pval for pkey='%s', lkey='%s'" \
% (pkey, lkey)
msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
print(msg)
print_exc(exc)
pval = 'N/A'
try:
if not filters:
if pkey == 'summary':
page += 'Summary information:'
elif pval and pval != 'N/A':
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
args = make_args(lkey, pval, inst)
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (args, pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if 'link' in item:
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = '<a href="/das/download?lfn=%s">Download</a>'\
% lfn if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
links.append(self.templatepage(\
'xsecdb', primds=path.split('/')[1]))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except Exception as exc:
print_exc(exc)
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
if filters and not aggrtrs:
page += add_filter_values(row, filters)
else:
page += adjust_values(func, gen, links, pkey)
pad = ""
try:
if 'das' in row and 'system' in row['das']:
systems = self.systems(row['das']['system'])
else:
systems = "" # no das record
print(dastimestamp('DAS ERROR '), \
'record without DAS key', row)
except KeyError as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(dasquery, row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if 'das' in row and 'conflict' in row['das']:
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
hints = ''
for hint in row.get('hints', {}):
if hint:
hints += self.templatepage('hint',
hint=hint, base=self.base, dbs=self.dbs_global)
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict, hints=hints)
page += '</div>'
old = row
main += fltpage
if dup and not dasquery.aggregators:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
if total>10:
main += '<hr class="line" />'
main += pager
main += '<hr class="line" />'
proc_time = self.processing_time(dasquery)
if proc_time:
msg = 'processing time: %5.3f sec, ' % proc_time
else:
msg = ''
msg += 'cache server time: %5.3f sec' % head['ctime']
main += '<div align="right">%s</div>' % msg
return main