def phedex_files(phedex_url, kwds):
"Get file information from Phedex"
params = dict(kwds) # parameters to be send to Phedex
site = kwds.get('site', None)
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
params.pop('site')
elif site and se_pattern.match(site):
params.update({'se': site})
params.pop('site')
else:
return
expire = 600 # set some expire since we're not going to use it
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_url, params, headers, expire, ckey=CKEY, cert=CERT,
system='phedex')
tags = 'block.file.name'
prim_key = 'block'
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
files = ddict.get('block.file')
if not isinstance(files, list):
files = [files]
for row in files:
yield row['name']
def test_get(self):
"""test get method"""
rec = DotDict(self.rec1)
expect = [1, 2]
result = rec.get('a.c')
self.assertEqual(expect, result)
self.assertEqual(expect, rec['a.c'])
def test_get(self):
"""test get method"""
rec = DotDict(self.rec1)
expect = [1,2]
result = rec.get('a.c')
self.assertEqual(expect, result)
self.assertEqual(expect, rec['a.c'])
def phedex_files(phedex_url, kwds):
"Get file information from Phedex"
params = dict(kwds) # parameters to be send to Phedex
site = kwds.get('site', None)
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
params.pop('site')
elif site and se_pattern.match(site):
params.update({'se': site})
params.pop('site')
else:
return
expire = 600 # set some expire since we're not going to use it
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_url, params, headers, expire, ckey=CKEY, cert=CERT,
system='phedex')
tags = 'block.file.name'
prim_key = 'block'
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
files = ddict.get('block.file')
if not isinstance(files, list):
files = [files]
for row in files:
yield row['name']
def fltpage(self, row):
"""Prepare filter snippet for a given query"""
rowkeys = []
page = ''
if row and row.has_key('das') and row['das'].has_key('primary_key'):
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
lmax = len(row[mkey]) if len(row[mkey]) < 5 else 5
sublist = [row[mkey][i] for i in range(0, lmax)]
ndict = DotDict({mkey:sublist})
rowkeys = [k for k in ndict.get_keys(mkey)]
else:
rowkeys = [k for k in DotDict(row).get_keys(mkey)]
rowkeys.sort()
rowkeys += ['das.conflict']
dflt = das_filters() + das_aggregators()
dflt.remove('unique')
page = self.templatepage('das_filters', \
filters=dflt, das_keys=rowkeys)
except Exception as exc:
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print msg
print_exc(exc)
pass
return page
def filter_with_filters(rows, filters):
"""
Filter given rows with provided set of filters.
"""
for row in rows:
ddict = DotDict(row)
flist = [(f, ddict.get(f)) for f in filters]
for idx in flist:
yield idx
def site4dataset(dbs_url, phedex_api, args, expire):
"Yield site information about given dataset"
# DBS part
dataset = args['dataset']
try:
totblocks, totfiles = dataset_summary(dbs_url, dataset)
except Exception as err:
error = str(err)
reason = "Can't find #block, #files info in DBS for dataset=%s" \
% dataset
yield {'site': {'error': error, 'reason': reason}}
return
# Phedex part
phedex_args = {'dataset':args['dataset']}
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_api, phedex_args, headers, expire, post=True,
system='phedex')
prim_key = 'block'
tags = 'block.replica.node'
site_info = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
replicas = ddict.get('block.replica')
if not isinstance(replicas, list):
replicas = [replicas]
for row in replicas:
if not row or 'node' not in row:
continue
node = row['node']
files = int(row['files'])
complete = 1 if row['complete'] == 'y' else 0
if node in site_info:
files = site_info[node]['files'] + files
nblks = site_info[node]['blocks'] + 1
bc_val = site_info[node]['blocks_complete']
b_complete = bc_val+1 if complete else bc_val
else:
b_complete = 1 if complete else 0
nblks = 1
site_info[node] = {'files': files, 'blocks': nblks,
'blocks_complete': b_complete}
row = {}
for key, val in site_info.iteritems():
if totfiles:
nfiles = '%5.2f%%' % (100*float(val['files'])/totfiles)
else:
nfiles = 'N/A'
if totblocks:
nblks = '%5.2f%%' % (100*float(val['blocks'])/totblocks)
else:
nblks = 'N/A'
ratio = float(val['blocks_complete'])/val['blocks']
b_completion = '%5.2f%%' % (100*ratio)
row = {'site':{'name':key, 'dataset_fraction': nfiles,
'block_fraction': nblks, 'block_completion': b_completion}}
yield row
def test_set(self):
"""test set method"""
rec = DotDict(self.rec1)
rec['a.d'] = 111
self.assertEqual(rec['a.d'], 111)
self.assertRaises(Exception, rec.set, ('a.c.d', 1))
rec = DotDict(self.rec2)
rec['a.b'] = 111
self.assertEqual(rec['a.b'], 111)
def test_DotDict(self):
"""Test DotDict class"""
res = {u'zip' : {u'code':u'14850'}}
mdict = DotDict(res)
mdict['zip.code'] = 14850
expect = {u'zip' : {u'code':14850}}
self.assertEqual(expect, mdict)
res = {'a':{'b':{'c':10}, 'd':10}}
mdict = DotDict(res)
mdict['x.y.z'] = 10
expect = {'a':{'b':{'c':10}, 'd':10}, 'x':{'y':{'z':10}}}
self.assertEqual(expect, mdict)
mdict['a.b.k.m'] = 10
expect = {'a':{'b':{'c':10, 'k':{'m':10}}, 'd':10}, 'x':{'y':{'z':10}}}
self.assertEqual(expect, mdict)
expect = 10
result = mdict.get('a.b.k.m')
self.assertEqual(expect, result)
res = {'a':{'b':{'c':10}, 'd':[{'x':1}, {'x':2}]}}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.d.x')
self.assertEqual(expect, result)
expect = None
result = mdict.get('a.M.Z')
self.assertEqual(expect, result)
res = {'a': {'b': {'c':1, 'd':2}}}
mdict = DotDict(res)
expect = {'a': {'b': {'c':1}}}
mdict.delete('a.b.d')
self.assertEqual(expect, mdict)
def test_DotDict_list(self):
"""Test DotDict class"""
res = {'a':[{'b':1, 'c':1}, {'c':1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
res = {'a':[{'c':1}, {'b':1, 'c':1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {'a':[{'b':1, 'c':1}, {'c':2}]}
mdict = DotDict(res)
expect = ['a.b', 'a.c']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
res = {'a':[{'b': [{'c':2}, {'c':{'d':1}}]},
{'b': [{'c':4}, {'c':5}]}]}
mdict = DotDict(res)
expect = ['a.b', 'a.b.c', 'a.b.c.d']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
def fix_times(row):
"Convert creation/modification times into DAS time format"
rec = DotDict(row)
times = ['creation_time', 'modification_time', 'create_time', 'end_time']
def callback(elem, key):
# print "\n### callback", key, elem
val = elem[key]
if val:
elem[key] = presentation_datetime(val)
for key in rec.get_keys():
if key.find('creation_time') != -1 or \
key.find('modification_time') != -1 or \
key.find('start_time') != -1 or \
key.find('end_time') != -1:
rec.set_values(key, callback)
def tableview(self, head, data):
"""
Represent data in tabular view.
"""
kwargs = head.get('args')
total = head.get('nresults', 0)
dasquery = kwargs['dasquery']
filters = dasquery.filters
titles = []
apilist = head.get('apilist')
page = self.pagination(head)
if filters:
for flt in filters:
if flt.find('=') != -1 or flt.find('>') != -1 or \
flt.find('<') != -1:
continue
titles.append(flt)
style = 1
tpage = ""
pkey = None
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
rec = []
if not pkey and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key'].split('.')[0]
if dasquery.filters:
for flt in dasquery.filters:
rec.append(DotDict(row).get(flt))
else:
titles = []
for key, val in row.items():
skip = 0
if not filters:
if key in titles:
skip = 1
else:
titles.append(key)
if not skip:
rec.append(val)
if style:
style = 0
else:
style = 1
link = '<a href="/das/records/%s?collection=merge">link</a>' \
% quote(str(row['_id'])) # cgi.escape the id
tpage += self.templatepage('das_table_row', rec=rec, tag='td', \
style=style, encode=1, record=link)
theads = list(titles) + ['Record']
thead = self.templatepage('das_table_row', rec=theads, tag='th', \
style=0, encode=0, record=0)
page += '<br />'
page += '<table class="das_table">' + thead + tpage + '</table>'
page += '<br />'
page += '<div align="right">DAS cache server time: %5.3f sec</div>' \
% head['ctime']
return page
def test_aggregators(self):
"""test aggregators dict records"""
rows = []
data = {
'block': {
'name': 'AAA',
'replica': [{
'name': 'a',
'size': 1
}, {
'name': 'b',
'size': 10
}]
}
}
rows.append(data)
data = {
'block': {
'name': 'AAA',
'replica': [{
'name': 'a',
'size': 2
}, {
'name': 'b',
'size': 20
}]
}
}
rows.append(data)
expect = 33
robj = das_func('sum', 'block.replica.size', rows)
self.assertEqual(expect, robj.result)
expect = 4
robj = das_func('count', 'block.replica.size', rows)
self.assertEqual(expect, robj.result)
expect = 1
robj = das_func('min', 'block.replica.size', rows)
self.assertEqual(expect, robj.result)
expect = 20
robj = das_func('max', 'block.replica.size', rows)
self.assertEqual(expect, robj.result)
expect = (1 + 10 + 2 + 20) / 4.
robj = das_func('avg', 'block.replica.size', rows)
self.assertEqual(expect, float(robj.result) / robj.rec_count)
expect = (10 + 2) // 2
robj = das_func('median', 'block.replica.size', rows)
val = (robj.result[len(robj.result)//2-1] + \
robj.result[len(robj.result)//2] )//2
self.assertEqual(expect, val)
expect = 20
drows = [DotDict(row) for row in rows]
robj = das_func('max', 'block.replica.size', drows)
self.assertEqual(expect, robj.result)
def makepy(self, dataset, instance):
"""
Request to create CMSSW py snippet for a given dataset
"""
pat = re.compile('/.*/.*/.*')
if not pat.match(dataset):
msg = 'Invalid dataset name'
return self.error(msg)
query = "file dataset=%s instance=%s | grep file.name" \
% (dataset, instance)
try:
data = self.dasmgr.result(query, idx=0, limit=0)
except Exception as exc:
print_exc(exc)
msg = 'Exception: %s\n' % str(exc)
msg += 'Unable to retrieve data for query=%s' % query
return self.error(msg)
lfns = []
for rec in data:
filename = DotDict(rec).get('file.name')
if filename not in lfns:
lfns.append(filename)
page = self.templatepage('das_files_py', lfnList=lfns, pfnList=[], isinstance=isinstance, list=list)
cherrypy.response.headers['Content-Type'] = "text/plain"
return page
def test_DotDict_values(self):
"""Test DotDict get_values method"""
res = {'a':[{'b':1, 'c':1}, {'c':2}]}
mdict = DotDict(res)
expect = [1]
result = [r for r in mdict.get_values('a.b')]
self.assertEqual(expect, result)
expect = [1,2]
result = [r for r in mdict.get_values('a.c')]
self.assertEqual(expect, result)
res = {'a':[{'b': [{'c':2}, {'c':3}]}, {'b': [{'c':4}, {'c':5}]}]}
mdict = DotDict(res)
expect = [2,3,4,5]
result = [r for r in mdict.get_values('a.b.c')]
self.assertEqual(expect, result)
def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]}
mdict = DotDict(res)
expect = ['a.b', 'a.c']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
res = {
'a': [{
'b': [{
'c': 2
}, {
'c': {
'd': 1
}
}]
}, {
'b': [{
'c': 4
}, {
'c': 5
}]
}]
}
mdict = DotDict(res)
expect = ['a.b', 'a.b.c', 'a.b.c.d']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
def plainview(self, head, data):
"""
Represent data in DAS plain view for queries with filters.
"""
dasquery = head['dasquery']
fields = dasquery.mongo_query.get('fields', [])
filters = [f for f in dasquery.filters if f != 'unique' and \
f.find('=') == -1 and f.find('<') == -1 and \
f.find('>') == -1]
results = ""
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
results += 'ERROR: %s' % reason
lookup_items = [i for i in fields if i not in das_record_keys()]
for row in data:
if filters:
for flt in filters:
try:
for obj in DotDict(row).get_values(flt):
results += str(obj) + ' '
except:
pass
results += '\n'
else:
for item in lookup_items:
if item != lookup_items[0]:
results += ', '
try:
systems = row['das']['system']
mapkey = self.dasmapping.find_mapkey(systems[0], item)
if not mapkey:
mapkey = '%s.name' % item
key, att = mapkey.split('.')
if key in row:
val = row[key]
if isinstance(val, dict):
results += val.get(att, '')
elif isinstance(val, list):
results += \
' '.join(set([str(i.get(att, '')) for i in val]))
except:
pass
results += '\n'
# use DAS sort_rows function instead of python set, since we need to
# preserve the order of records in final output
rows = [r for r in results.split('\n') if r]
results = '\n'.join([r for r in sort_rows(rows)])
return results
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ['a', 'a.b', 'a.c']
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
doc = {
"site": [{
"mapping": {
"pfn": [{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}, {
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}]
}
}]
}
result = DotDict(doc).get_keys()
result.sort()
expect = [
'site', 'site.mapping', 'site.mapping.pfn',
'site.mapping.pfn.path', 'site.mapping.pfn.protocol',
'site.mapping.pfn.result'
]
self.assertEqual(expect, result)
def get_result_fieldlist(self, row):
rowkeys = []
if row and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if mkey not in row:
return []
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
ndict = DotDict({mkey: row[mkey][:10]})
rowkeys = list(ndict.get_keys(mkey))
else:
rowkeys = list(DotDict(row).get_keys(mkey))
rowkeys.sort()
rowkeys += ['das.conflict']
except Exception as exc:
# TODO: pkey.split fail only if called on non-string
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print(msg)
print_exc(exc)
return rowkeys
def get_result_fieldlist(self, row):
rowkeys = []
if row and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if mkey not in row:
return []
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
ndict = DotDict({mkey: row[mkey][:10]})
rowkeys = list(ndict.get_keys(mkey))
else:
rowkeys = list(DotDict(row).get_keys(mkey))
rowkeys.sort()
rowkeys += ['das.conflict']
except Exception as exc:
# TODO: pkey.split fail only if called on non-string
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print msg
print_exc(exc)
return rowkeys
def repr_values(row, flt):
"Represent values of given row/filter in appropriate form"
values = [r for r in DotDict(row).get_values(flt)]
if not len(values):
val = ''
elif len(values) == 1:
val = repr_val(values[0])
else:
if isinstance(values[0], dict) or isinstance(values[0], list):
val = repr_val(values)
else:
val = ', '.join(set([str(v) for v in values]))
if flt.lower() == 'run.run_number':
if isinstance(val, basestring):
val = int(val.split('.')[0])
return val
def kws_js(dascore, query, idx, limit, jsfile, verbose=False):
"Write result of a given query into KWS js file"
print("Create: %s" % jsfile)
results = dascore.result(query, idx, limit)
tstamp = long(time.time())
with open(jsfile, 'a') as stream:
for row in results:
pkey = row['das']['primary_key']
ddict = DotDict(row)
value = ddict[pkey]
if value == '*' or value == 'null' or not value:
continue
jsrow = json.dumps(dict(value=value, ts=tstamp))
if verbose:
print(jsrow)
stream.write(jsrow)
stream.write('\n')
def test_DotDict_list(self):
"""Test DotDict class"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
res = {'a': [{'c': 1}, {'b': 1, 'c': 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
def test_DotDict_values(self):
"""Test DotDict get_values method"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]}
mdict = DotDict(res)
expect = [1]
result = [r for r in mdict.get_values('a.b')]
self.assertEqual(expect, result)
expect = [1, 2]
result = [r for r in mdict.get_values('a.c')]
self.assertEqual(expect, result)
res = {'a': [{'b': [{'c': 2}, {'c': 3}]}, {'b': [{'c': 4}, {'c': 5}]}]}
mdict = DotDict(res)
expect = [2, 3, 4, 5]
result = [r for r in mdict.get_values('a.b.c')]
self.assertEqual(expect, result)
def plainview(self, head, data):
"""
Represent data in DAS plain view for queries with filters.
"""
dasquery = head['dasquery']
fields = dasquery.mongo_query.get('fields', [])
filters = dasquery.filters
results = ""
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
results += 'ERROR: %s' % reason
for row in data:
if filters:
for flt in filters:
if flt.find('=') != -1 or flt.find('>') != -1 or \
flt.find('<') != -1:
continue
try:
for obj in DotDict(row).get_values(flt):
results += str(obj) + '\n'
except:
pass
results += '\n'
else:
for item in fields:
try:
mapkey = '%s.name' % item
key, att = mapkey.split('.')
if key in row:
val = row[key]
if isinstance(val, dict):
results += val.get(att, '')
elif isinstance(val, list):
for item in val:
results += item.get(att, '')
results += '\n'
except:
pass
results += '\n'
return results
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ['a', 'a.b', 'a.c']
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
doc = {"site": [{"mapping": { "pfn": [
{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
},
{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}]
}}]
}
result = DotDict(doc).get_keys()
result.sort()
expect = ['site',
'site.mapping',
'site.mapping.pfn',
'site.mapping.pfn.path',
'site.mapping.pfn.protocol',
'site.mapping.pfn.result']
self.assertEqual(expect, result)
def test_delete(self):
"""test delete method"""
rec = DotDict(self.rec1)
rec.delete('a.c')
expect = {'a':{'b':1}}
self.assertEqual(expect, rec)
def test_get_values(self):
"""test get_values method"""
rec = DotDict(self.rec2)
expect = [1, 2, 10, 20]
result = [o for o in rec.get_values('a.c')]
self.assertEqual(expect, result)
def listview(self, head, data):
"""
Represent data in list view.
"""
kwargs = head.get('args')
uinput = kwargs.get('input', '')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery', None)
if not dasquery:
inst = head.get('instance', self.dbs_global)
dasquery = DASQuery(uinput, instance=inst)
inst = dasquery.instance
filters = dasquery.filters
aggrtrs = dasquery.aggregators
pager = self.pagination(head)
main = pager
style = 'white'
rowkeys = []
fltpage = self.filter_bar(dasquery)
page = ''
old = None
dup = False
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not row:
continue
if not dup and old and identical_data_records(old, row):
dup = True
error = row.get('error', None)
try:
mongo_id = row['_id']
except Exception as exc:
msg = 'Exception: %s\n' % str(exc)
msg += 'Fail to process row\n%s' % str(row)
raise Exception(msg)
page += '<div class="%s"><hr class="line" />' % style
links = []
pkey = None
pval = None
lkey = None
if 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and not rowkeys and not fltpage:
fltpage = self.fltpage(dasquery)
try:
lkey = pkey.split('.')[0]
if pkey == 'summary':
pval = row[pkey]
else:
pval = [i for i in DotDict(row).get_values(pkey)]
if isinstance(pval, list):
if pval and not isinstance(pval[0], list):
pval = list(set(pval))
else:
pval = list(set(pval))
if len(pval) == 1:
pval = pval[0]
if pkey == 'run.run_number' or pkey == 'lumi.number':
if isinstance(pval, basestring):
pval = int(pval)
except Exception as exc:
msg = "Fail to extract pval for pkey='%s', lkey='%s'" \
% (pkey, lkey)
msg += "\npval='%s', type(pval)='%s'" % (pval, type(pval))
print(msg)
print_exc(exc)
pval = 'N/A'
try:
if not filters:
if pkey == 'summary':
page += 'Summary information:'
elif pval and pval != 'N/A':
page += '%s: ' % lkey.capitalize()
if lkey == 'parent' or lkey == 'child':
if str(pval).find('.root') != -1:
lkey = 'file'
else:
lkey = 'dataset'
if lkey in not_to_link():
page += '%s' % pval
elif isinstance(pval, list):
page += ', '.join(['<span class="highlight>"'+\
'<a href="/das/request?%s">%s</a></span>'\
% (make_args(lkey, i, inst), i) for i in pval])
else:
args = make_args(lkey, pval, inst)
page += '<span class="highlight">'+\
'<a href="/das/request?%s">%s</a></span>'\
% (args, pval)
else:
page += '%s: N/A' % lkey.capitalize()
plist = self.dasmgr.mapping.presentation(lkey)
linkrec = None
for item in plist:
if 'link' in item:
linkrec = item['link']
break
if linkrec and pval and pval != 'N/A' and \
not isinstance(pval, list) and not error:
links += [l for l in make_links(linkrec, pval, inst)]
if pkey and pkey == 'file.name':
try:
lfn = DotDict(row).get('file.name')
val = '<a href="/das/download?lfn=%s">Download</a>'\
% lfn if lfn else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'site.name':
try:
site = DotDict(row).get('site.name')
val = self.templatepage(\
'sitedb', item=site, api="sites") if site else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'user.name':
try:
user = DotDict(row).get('user.username')
val = self.templatepage(\
'sitedb', item=user, api="people") if user else ''
if val: links.append(val)
except:
pass
if pkey and pkey == 'dataset.name':
try:
path = DotDict(row).get('dataset.name')
if path:
links.append(self.templatepage(\
'makepy', path=path, inst=inst))
if inst == self.dbs_global:
links.append(self.templatepage(\
'phedex_subscription', path=path))
links.append(self.templatepage(\
'xsecdb', primds=path.split('/')[1]))
except:
pass
if pkey and pkey == 'release.name':
rel = '["%s"]' % DotDict(row).get('release.name')
url = 'https://cmstags.cern.ch/tc/py_getReleasesTags?'
url += 'diff=false&releases=%s' % urllib.quote(rel)
links.append('<a href="%s">Packages</a>' % url)
except Exception as exc:
print_exc(exc)
pval = 'N/A'
gen = self.convert2ui(row, pkey)
if self.dasmgr:
func = self.dasmgr.mapping.daskey_from_presentation
if filters and not aggrtrs:
page += add_filter_values(row, filters)
else:
page += adjust_values(func, gen, links, pkey)
pad = ""
try:
if 'das' in row and 'system' in row['das']:
systems = self.systems(row['das']['system'])
else:
systems = "" # no das record
print(dastimestamp('DAS ERROR '), \
'record without DAS key', row)
except KeyError as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
except Exception as exc:
print_exc(exc)
systems = "" # we don't store systems for aggregated records
jsonhtml = das_json(dasquery, row, pad)
jsonhtml = jsonhtml.replace(\
'request?', 'request?instance=%s&' % inst)
if not links:
page += '<br />'
if 'das' in row and 'conflict' in row['das']:
conflict = ', '.join(row['das']['conflict'])
else:
conflict = ''
hints = ''
for hint in row.get('hints', {}):
if hint:
hints += self.templatepage('hint',
hint=hint, base=self.base, dbs=self.dbs_global)
page += self.templatepage('das_row', systems=systems, \
sanitized_data=jsonhtml, id=mongo_id, rec_id=mongo_id,
conflict=conflict, hints=hints)
page += '</div>'
old = row
main += fltpage
if dup and not dasquery.aggregators:
main += self.templatepage('das_duplicates', uinput=uinput,
instance=inst)
main += page
if total>10:
main += '<hr class="line" />'
main += pager
main += '<hr class="line" />'
proc_time = self.processing_time(dasquery)
if proc_time:
msg = 'processing time: %5.3f sec, ' % proc_time
else:
msg = ''
msg += 'cache server time: %5.3f sec' % head['ctime']
main += '<div align="right">%s</div>' % msg
return main
def tableview(self, head, data):
"""
Represent data in tabular view.
"""
kwargs = head.get('args')
total = head.get('nresults', 0)
apilist = head.get('apilist')
dasquery = head.get('dasquery')
filters = dasquery.filters
sdir = getarg(kwargs, 'dir', '')
titles = []
page = self.pagination(head)
fltbar = self.filter_bar(dasquery)
if filters:
for flt in filters:
if flt.find('=') != -1 or flt.find('>') != -1 or \
flt.find('<') != -1:
continue
titles.append(flt)
style = 1
tpage = ""
pkey = None
status = head.get('status', None)
if status == 'fail':
reason = head.get('reason', '')
if reason:
page += '<br/><span class="box_red">%s</span>' % reason
for row in data:
if not fltbar:
fltbar = self.fltpage(dasquery)
try: # we don't need to show qhash in table view
del row['qhash']
except:
pass
rec = []
if not pkey and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key'].split('.')[0]
if filters:
for flt in filters:
rec.append(DotDict(row).get(flt))
else:
gen = self.convert2ui(row)
titles = []
for uikey, val, _link, _desc, _examples in gen:
skip = 0
if not filters:
if uikey in titles:
skip = 1
else:
titles.append(uikey)
if not skip:
rec.append(val)
if style:
style = 0
else:
style = 1
link = '<a href="/das/records/%s?collection=merge">link</a>' \
% quote(str(row['_id'])) # cgi.escape the id
tpage += self.templatepage('das_table_row', rec=rec, tag='td', \
style=style, encode=1, record=link)
if sdir == 'asc':
sdir = 'desc'
elif sdir == 'desc':
sdir = 'asc'
else: # default sort direction
sdir = 'asc'
theads = []
for title in titles:
theads.append(title)
theads.append('Record')
thead = self.templatepage('das_table_row', rec=theads, tag='th', \
style=0, encode=0, record=0)
page += fltbar
page += '<br />'
page += '<table class="das_table">' + thead + tpage + '</table>'
page += '<br />'
page += '<div align="right">DAS cache server time: %5.3f sec</div>' \
% head['ctime']
return page
def helper(self, api, args, expire):
"""
Class helper function which yields results for given
set of input parameters. It yeilds the data record which
must contain combined attribute corresponding to systems
used to produce record content.
"""
dbs_url = self.map[api]['services'][self.dbs]
phedex_url = self.map[api]['services']['phedex']
# make phedex_api from url, but use xml version for processing
phedex_api = phedex_url.replace('/json/', '/xml/') + '/blockReplicas'
if api == 'dataset4site_release' or \
api == 'dataset4site_release_parent' or \
api == 'child4site_release_dataset':
# DBS part
datasets = set()
release = args['release']
parent = args.get('parent', None)
for row in dbs_dataset4release_parent(dbs_url, release, parent):
datasets.add(row)
# Phedex part
if args['site'].find('.') != -1: # it is SE
phedex_args = {
'dataset': list(datasets),
'se': '%s' % args['site']
}
else:
phedex_args = {
'dataset': list(datasets),
'node': '%s*' % args['site']
}
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_api, phedex_args, headers, expire, system='phedex')
prim_key = 'block'
tags = 'block.replica.node'
found = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
block = ddict.get('block.name')
bbytes = ddict.get('block.bytes')
files = ddict.get('block.files')
found_dataset = block.split('#')[0]
if found_dataset in found:
val = found[found_dataset]
found[found_dataset] = {
'bytes': val['bytes'] + bbytes,
'files': val['files'] + files
}
else:
found[found_dataset] = {'bytes': bbytes, 'files': files}
for name, val in found.items():
record = dict(name=name, size=val['bytes'], files=val['files'])
if api == 'child4site_release_dataset':
yield {'child': record}
else:
yield {'dataset': record}
del datasets
del found
if api == 'site4dataset':
try:
gen = site4dataset(dbs_url, phedex_api, args, expire)
for row in gen:
sname = row.get('site', {}).get('name', '')
skind = self.site_info(phedex_url, sname)
row['site'].update({'kind': skind})
yield row
except Exception as err:
print_exc(err)
tstamp = dastimestamp('')
msg = tstamp + ' Exception while processing DBS/Phedex info:'
msg += str(err)
row = {
'site': {
'name': 'Fail to look-up site info',
'error': msg,
'dataset_fraction': 'N/A',
'block_fraction': 'N/A',
'block_completion': 'N/A'
},
'error': msg
}
yield row
if api == 'files4dataset_runs_site' or \
api == 'files4block_runs_site':
run_value = args.get('run', [])
if isinstance(run_value, dict) and '$in' in run_value:
runs = run_value['$in']
elif isinstance(run_value, list):
runs = run_value
else:
if int_number_pattern.match(str(run_value)):
runs = [run_value]
else:
runs = []
args.update({'runs': runs})
files = dbs_find('file', dbs_url, args)
site = args.get('site')
phedex_api = phedex_url.replace('/json/',
'/xml/') + '/fileReplicas'
for fname in files4site(phedex_api, files, site):
yield {'file': {'name': fname}}
def site4dataset(dbs_url, phedex_api, args, expire):
"Yield site information about given dataset"
# DBS part
dataset = args['dataset']
try:
totblocks, totfiles = dataset_summary(dbs_url, dataset)
except Exception as err:
error = 'combined service unable to process your request'
reason = "Fail to parse #block, #files info, %s" % str(err)
yield {
'site': {
'name': 'N/A',
'se': 'N/A',
'error': error,
'reason': reason
}
}
return
# Phedex part
phedex_args = {'dataset': args['dataset']}
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_api, phedex_args, headers, expire, system='phedex')
prim_key = 'block'
tags = 'block.replica.node'
site_info = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
replicas = ddict.get('block.replica')
if not isinstance(replicas, list):
replicas = [replicas]
for row in replicas:
if not row or 'node' not in row:
continue
node = row['node']
files = int(row['files'])
complete = 1 if row['complete'] == 'y' else 0
if node in site_info:
files = site_info[node]['files'] + files
nblks = site_info[node]['blocks'] + 1
bc_val = site_info[node]['blocks_complete']
b_complete = bc_val + 1 if complete else bc_val
else:
b_complete = 1 if complete else 0
nblks = 1
site_info[node] = {
'files': files,
'blocks': nblks,
'blocks_complete': b_complete
}
row = {}
for key, val in site_info.items():
if totfiles:
nfiles = '%5.2f%%' % (100 * float(val['files']) / totfiles)
else:
nfiles = 'N/A'
if totblocks:
nblks = '%5.2f%%' % (100 * float(val['blocks']) / totblocks)
else:
nblks = 'N/A'
ratio = float(val['blocks_complete']) / val['blocks']
b_completion = '%5.2f%%' % (100 * ratio)
row = {
'site': {
'name': key,
'dataset_fraction': nfiles,
'block_fraction': nblks,
'block_completion': b_completion
}
}
yield row
def test_DotDict(self):
"""Test DotDict class"""
res = {u'zip': {u'code': u'14850'}}
mdict = DotDict(res)
mdict['zip.code'] = 14850
expect = {u'zip': {u'code': 14850}}
self.assertEqual(expect, mdict)
res = {'a': {'b': {'c': 10}, 'd': 10}}
mdict = DotDict(res)
mdict['x.y.z'] = 10
expect = {'a': {'b': {'c': 10}, 'd': 10}, 'x': {'y': {'z': 10}}}
self.assertEqual(expect, mdict)
mdict['a.b.k.m'] = 10
expect = {
'a': {
'b': {
'c': 10,
'k': {
'm': 10
}
},
'd': 10
},
'x': {
'y': {
'z': 10
}
}
}
self.assertEqual(expect, mdict)
expect = 10
result = mdict.get('a.b.k.m')
self.assertEqual(expect, result)
res = {'a': {'b': {'c': 10}, 'd': [{'x': 1}, {'x': 2}]}}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.d.x')
self.assertEqual(expect, result)
expect = None
result = mdict.get('a.M.Z')
self.assertEqual(expect, result)
res = {'a': {'b': {'c': 1, 'd': 2}}}
mdict = DotDict(res)
expect = {'a': {'b': {'c': 1}}}
mdict.delete('a.b.d')
self.assertEqual(expect, mdict)
def helper(self, api, args, expire):
"""
Class helper function which yields results for given
set of input parameters. It yeilds the data record which
must contain combined attribute corresponding to systems
used to produce record content.
"""
dbs_url = self.map[api]['services'][self.dbs]
phedex_url = self.map[api]['services']['phedex']
# make phedex_api from url, but use xml version for processing
phedex_api = phedex_url.replace('/json/', '/xml/') + '/blockReplicas'
if api == 'dataset4site_release' or \
api == 'dataset4site_release_parent' or \
api == 'child4site_release_dataset':
# DBS part
datasets = set()
release = args['release']
parent = args.get('parent', None)
for row in dbs_dataset4release_parent(dbs_url, release, parent):
datasets.add(row)
# Phedex part
if args['site'].find('.') != -1: # it is SE
phedex_args = {'dataset':list(datasets),
'se': '%s' % args['site']}
else:
phedex_args = {'dataset':list(datasets),
'node': '%s*' % args['site']}
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_api, phedex_args, headers, expire, system='phedex')
prim_key = 'block'
tags = 'block.replica.node'
found = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
block = ddict.get('block.name')
bbytes = ddict.get('block.bytes')
files = ddict.get('block.files')
found_dataset = block.split('#')[0]
if found_dataset in found:
val = found[found_dataset]
found[found_dataset] = {'bytes': val['bytes'] + bbytes,
'files': val['files'] + files}
else:
found[found_dataset] = {'bytes': bbytes, 'files': files}
for name, val in found.items():
record = dict(name=name, size=val['bytes'], files=val['files'])
if api == 'child4site_release_dataset':
yield {'child': record}
else:
yield {'dataset':record}
del datasets
del found
if api == 'site4block':
pass
if api == 'site4dataset':
try:
gen = site4dataset(dbs_url, phedex_api, args, expire)
for row in gen:
sname = row.get('site', {}).get('name', '')
skind = self.site_info(phedex_url, sname)
row['site'].update({'kind':skind})
yield row
except Exception as err:
print_exc(err)
tstamp = dastimestamp('')
msg = tstamp + ' Exception while processing DBS/Phedex info:'
msg += str(err)
row = {'site':{'name':'Fail to look-up site info',
'error':msg, 'dataset_fraction': 'N/A',
'block_fraction':'N/A', 'block_completion':'N/A'},
'error': msg}
yield row
if api == 'files4dataset_runs_site' or \
api == 'files4block_runs_site':
run_value = args.get('run', [])
if isinstance(run_value, dict) and '$in' in run_value:
runs = run_value['$in']
elif isinstance(run_value, list):
runs = run_value
else:
if int_number_pattern.match(str(run_value)):
runs = [run_value]
else:
runs = []
args.update({'runs': runs})
files = dbs_find('file', dbs_url, args)
site = args.get('site')
phedex_api = phedex_url.replace('/json/', '/xml/') + '/fileReplicas'
for fname in files4site(phedex_api, files, site):
yield {'file':{'name':fname}}
def test_delete(self):
"""test delete method"""
rec = DotDict(self.rec1)
rec.delete('a.c')
expect = {'a': {'b': 1}}
self.assertEqual(expect, rec)
def set_misses(self, dasquery, api, genrows):
"""
Check and adjust DAS records wrt input query. If some of the DAS
keys are missing, add it with its value to the DAS record.
"""
# look-up primary key
prim_key = self.dasmapping.primary_key(self.name, api)
# Scan all docs and store those whose size above MongoDB limit into
# GridFS
map_key = self.dasmapping.primary_mapkey(self.name, api)
genrows = parse2gridfs(self.gfs, map_key, genrows, self.logger)
spec = dasquery.mongo_query['spec']
row = next(genrows)
ddict = DotDict(row)
keys2adjust = []
for key in spec.keys():
val = ddict.get(key)
if spec[key] != val and key not in keys2adjust:
keys2adjust.append(key)
msg = "adjust keys %s" % keys2adjust
self.logger.debug(msg)
count = 0
if keys2adjust:
# adjust of the rows
for row in yield_rows(row, genrows):
ddict = DotDict(row)
pval = ddict.get(map_key)
if isinstance(pval, dict) and 'error' in pval:
ddict[map_key] = ''
ddict.update({prim_key: pval})
for key in keys2adjust:
value = spec[key]
existing_value = ddict.get(key)
# the way to deal with proximity/patern/condition results
if (isinstance(value, str) or isinstance(value, unicode))\
and value.find('*') != -1: # we got pattern
if existing_value:
value = existing_value
elif isinstance(value, dict) or \
isinstance(value, list): # we got condition
if existing_value:
value = existing_value
elif isinstance(value, dict) and \
'$in' in value: # we got a range {'$in': []}
value = value['$in']
elif isinstance(value, dict) and \
'$lte' in value and '$gte' in value:
# we got a between range
value = [value['$gte'], value['$lte']]
else:
value = json.dumps(value)
elif existing_value and value != existing_value:
# we got proximity results
if 'proximity' in ddict:
proximity = DotDict({key: existing_value})
ddict['proximity'].update(proximity)
else:
proximity = DotDict({})
proximity[key] = existing_value
ddict['proximity'] = proximity
else:
if existing_value:
value = existing_value
ddict[key] = value
yield ddict
count += 1
else:
yield row
for row in genrows:
yield row
count += 1
msg = "yield %s rows" % count
self.logger.debug(msg)
def test_get_values(self):
"""test get_values method"""
rec = DotDict(self.rec2)
expect = [1, 2, 10, 20]
result = [o for o in rec.get_values('a.c')]
self.assertEqual(expect, result)
def set_misses(self, dasquery, api, genrows):
"""
Check and adjust DAS records wrt input query. If some of the DAS
keys are missing, add it with its value to the DAS record.
"""
# look-up primary key
prim_key = self.dasmapping.primary_key(self.name, api)
# Scan all docs and store those whose size above MongoDB limit into
# GridFS
map_key = self.dasmapping.primary_mapkey(self.name, api)
genrows = parse2gridfs(self.gfs, map_key, genrows, self.logger)
spec = dasquery.mongo_query['spec']
row = next(genrows)
ddict = DotDict(row)
keys2adjust = []
for key in spec.keys():
val = ddict.get(key)
if spec[key] != val and key not in keys2adjust:
keys2adjust.append(key)
msg = "adjust keys %s" % keys2adjust
self.logger.debug(msg)
count = 0
if keys2adjust:
# adjust of the rows
for row in yield_rows(row, genrows):
ddict = DotDict(row)
pval = ddict.get(map_key)
if isinstance(pval, dict) and 'error' in pval:
ddict[map_key] = ''
ddict.update({prim_key: pval})
for key in keys2adjust:
value = spec[key]
existing_value = ddict.get(key)
# the way to deal with proximity/patern/condition results
if (isinstance(value, str) or isinstance(value, unicode))\
and value.find('*') != -1: # we got pattern
if existing_value:
value = existing_value
elif isinstance(value, dict) or \
isinstance(value, list): # we got condition
if existing_value:
value = existing_value
elif isinstance(value, dict) and \
'$in' in value: # we got a range {'$in': []}
value = value['$in']
elif isinstance(value, dict) and \
'$lte' in value and '$gte' in value:
# we got a between range
value = [value['$gte'], value['$lte']]
else:
value = json.dumps(value)
elif existing_value and value != existing_value:
# we got proximity results
if 'proximity' in ddict:
proximity = DotDict({key:existing_value})
ddict['proximity'].update(proximity)
else:
proximity = DotDict({})
proximity[key] = existing_value
ddict['proximity'] = proximity
else:
if existing_value:
value = existing_value
ddict[key] = value
yield ddict
count += 1
else:
yield row
for row in genrows:
yield row
count += 1
msg = "yield %s rows" % count
self.logger.debug(msg)
def helper(self, url, api, args, expire):
"""
Class helper function which yields results for given
set of input parameters. It yeilds the data record which
must contain combined attribute corresponding to systems
used to produce record content.
"""
dbs_url = url['dbs']
phedex_url = url['phedex']
if api == 'combined_dataset4site_release':
# DBS part
datasets = set()
for row in dbs_dataset4site_release(dbs_url, self.getdata, args['release']):
datasets.add(row)
# Phedex part
if args['site'].find('.') != -1: # it is SE
phedex_args = {'dataset':list(datasets),
'se': '%s' % args['site']}
else:
phedex_args = {'dataset':list(datasets),
'node': '%s*' % args['site']}
headers = {'Accept': 'text/xml'}
source, expire = \
self.getdata(phedex_url, phedex_args, expire, headers, post=True)
prim_key = 'block'
tags = 'block.replica.node'
found = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
block = ddict.get('block.name')
bbytes = ddict.get('block.bytes')
files = ddict.get('block.files')
found_dataset = block.split('#')[0]
if found.has_key(found_dataset):
val = found[found_dataset]
found[found_dataset] = {'bytes': val['bytes'] + bbytes,
'files': val['files'] + files}
else:
found[found_dataset] = {'bytes': bbytes, 'files': files}
for name, val in found.iteritems():
record = dict(name=name, size=val['bytes'], files=val['files'],
combined=['dbs', 'phedex'])
yield {'dataset':record}
del datasets
del found
if api == 'combined_site4dataset':
# DBS part
dataset = args['dataset']
totblocks, totfiles = \
dataset_summary(dbs_url, self.getdata, dataset)
# Phedex part
phedex_args = {'dataset':args['dataset']}
headers = {'Accept': 'text/xml'}
source, expire = \
self.getdata(phedex_url, phedex_args, expire, headers, post=True)
prim_key = 'block'
tags = 'block.replica.node'
found = {}
site_info = {}
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
replicas = ddict.get('block.replica')
if not isinstance(replicas, list):
replicas = [replicas]
for row in replicas:
if not row or not row.has_key('node'):
continue
node = row['node']
files = int(row['files'])
complete = 1 if row['complete'] == 'y' else 0
if site_info.has_key(node):
files = site_info[node]['files'] + files
nblks = site_info[node]['blocks'] + 1
bc_val = site_info[node]['blocks_complete']
b_complete = bc_val+1 if complete else bc_val
else:
b_complete = 1 if complete else 0
nblks = 1
site_info[node] = {'files': files, 'blocks': nblks,
'blocks_complete': b_complete}
row = {}
for key, val in site_info.iteritems():
if totfiles:
nfiles = '%5.2f%%' % (100*float(val['files'])/totfiles)
else:
nfiles = 'N/A'
if totblocks:
nblks = '%5.2f%%' % (100*float(val['blocks'])/totblocks)
else:
nblks = 'N/A'
ratio = float(val['blocks_complete'])/val['blocks']
b_completion = '%5.2f%%' % (100*ratio)
row = {'site':{'name':key, 'dataset_fraction': nfiles,
'block_fraction': nblks, 'block_completion': b_completion}}
yield row
