def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]}
mdict = DotDict(res)
expect = ['a.b', 'a.c']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
res = {
'a': [{
'b': [{
'c': 2
}, {
'c': {
'd': 1
}
}]
}, {
'b': [{
'c': 4
}, {
'c': 5
}]
}]
}
mdict = DotDict(res)
expect = ['a.b', 'a.b.c', 'a.b.c.d']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {'a':[{'b':1, 'c':1}, {'c':2}]}
mdict = DotDict(res)
expect = ['a.b', 'a.c']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
res = {'a':[{'b': [{'c':2}, {'c':{'d':1}}]},
{'b': [{'c':4}, {'c':5}]}]}
mdict = DotDict(res)
expect = ['a.b', 'a.b.c', 'a.b.c.d']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
def fltpage(self, row):
"""Prepare filter snippet for a given query"""
rowkeys = []
page = ''
if row and row.has_key('das') and row['das'].has_key('primary_key'):
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
lmax = len(row[mkey]) if len(row[mkey]) < 5 else 5
sublist = [row[mkey][i] for i in range(0, lmax)]
ndict = DotDict({mkey:sublist})
rowkeys = [k for k in ndict.get_keys(mkey)]
else:
rowkeys = [k for k in DotDict(row).get_keys(mkey)]
rowkeys.sort()
rowkeys += ['das.conflict']
dflt = das_filters() + das_aggregators()
dflt.remove('unique')
page = self.templatepage('das_filters', \
filters=dflt, das_keys=rowkeys)
except Exception as exc:
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print msg
print_exc(exc)
pass
return page
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ['a', 'a.b', 'a.c']
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
doc = {
"site": [{
"mapping": {
"pfn": [{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}, {
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}]
}
}]
}
result = DotDict(doc).get_keys()
result.sort()
expect = [
'site', 'site.mapping', 'site.mapping.pfn',
'site.mapping.pfn.path', 'site.mapping.pfn.protocol',
'site.mapping.pfn.result'
]
self.assertEqual(expect, result)
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ['a', 'a.b', 'a.c']
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
doc = {"site": [{"mapping": { "pfn": [
{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
},
{
"protocol": "direct",
"result": "/store/$1",
"path": "/+hadoop/cms/store/(.*)"
}]
}}]
}
result = DotDict(doc).get_keys()
result.sort()
expect = ['site',
'site.mapping',
'site.mapping.pfn',
'site.mapping.pfn.path',
'site.mapping.pfn.protocol',
'site.mapping.pfn.result']
self.assertEqual(expect, result)
def fix_times(row):
"Convert creation/modification times into DAS time format"
rec = DotDict(row)
times = ['creation_time', 'modification_time', 'create_time', 'end_time']
def callback(elem, key):
# print "\n### callback", key, elem
val = elem[key]
if val:
elem[key] = presentation_datetime(val)
for key in rec.get_keys():
if key.find('creation_time') != -1 or \
key.find('modification_time') != -1 or \
key.find('start_time') != -1 or \
key.find('end_time') != -1:
rec.set_values(key, callback)
def get_result_fieldlist(self, row):
rowkeys = []
if row and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if mkey not in row:
return []
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
ndict = DotDict({mkey: row[mkey][:10]})
rowkeys = list(ndict.get_keys(mkey))
else:
rowkeys = list(DotDict(row).get_keys(mkey))
rowkeys.sort()
rowkeys += ['das.conflict']
except Exception as exc:
# TODO: pkey.split fail only if called on non-string
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print msg
print_exc(exc)
return rowkeys
def get_result_fieldlist(self, row):
rowkeys = []
if row and 'das' in row and 'primary_key' in row['das']:
pkey = row['das']['primary_key']
if pkey and (isinstance(pkey, str) or isinstance(pkey, unicode)):
try:
mkey = pkey.split('.')[0]
if mkey not in row:
return []
if isinstance(row[mkey], list):
# take first five or less entries from the list to cover
# possible aggregated records and extract row keys
ndict = DotDict({mkey: row[mkey][:10]})
rowkeys = list(ndict.get_keys(mkey))
else:
rowkeys = list(DotDict(row).get_keys(mkey))
rowkeys.sort()
rowkeys += ['das.conflict']
except Exception as exc:
# TODO: pkey.split fail only if called on non-string
msg = "Fail to pkey.split('.') for pkey=%s" % pkey
print(msg)
print_exc(exc)
return rowkeys
