class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
{"ui": "Block size", "das": "block.size"}]}, "type": "presentation"}
self.coll.insert(self.pmap)
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
dformat = 'JSON'
system = 'dbs3'
expire = 100
rec = {'system':system, 'urn': 'acquisitioneras', 'format':dformat,
'instances': ['prod/global'],
'url':url, 'expire': expire, 'lookup': 'era',
'params' : {},
'das_map': [
{"das_key": "era", "rec_key":"era.name", "api_arg":"era"}
],
'type': 'service'
}
self.coll.insert(rec)
ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
rec = {'verification_token':ver_token, 'type':'verification_token'}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
system = 'dbs3'
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
dformat = 'JSON'
expire = 100
instances = ["prod/global", "prod/phys01"]
api = 'primarydatasets'
params = {"primary_ds_name":"*"}
rec = {'system':system, 'urn':api, 'format':dformat, 'url':url,
'instances': instances,
'params': params, 'expire':expire, 'lookup': 'primary_dataset', 'wild_card':'*',
'das_map' : [dict(das_key='primary_dataset',
rec_key='primary_dataset.name',
api_arg='primary_dataset')],
'type': 'service'
}
self.mgr.add(rec)
smap = {api: {'url':url, 'expire':expire, 'keys': ['primary_dataset'],
'format': dformat, 'wild_card':'*', 'cert':None, 'ckey': None,
'services': '', 'lookup': 'primary_dataset',
'params': params }
}
rec = {'system':system, 'urn': 'datasetaccesstypes', 'format':dformat,
'instances': instances,
'url':url, 'expire': expire, 'lookup': 'status',
'params' : {'status':'*'},
'das_map': [
{"das_key": "status", "rec_key":"status.name", "api_arg":"status"}
],
'type': 'service'
}
self.mgr.add(rec)
api = 'datasetaccesstypes'
daskey = 'status'
rec_key = 'status.name'
api_input = 'status'
res = self.mgr.list_systems()
self.assertEqual([system], res)
res = self.mgr.list_apis()
res.sort()
self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ''
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {'system':system, 'type': 'notation',
'notations':[
{'api_output':'storage_element_name', 'rec_key':'se', 'api':''},
{'api_output':'number_of_events', 'rec_key':'nevents', 'api':''},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(['status'], res)
# build service map
smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
'keys': ['status'], 'format':dformat, 'wild_card':'*',
'services': '', 'lookup': daskey,
'params': {"status": "*"}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
system = "test"
rec = {'notations': [
{"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "admin.contacts.cms_name", "rec_key":"site.name", "api":""}
], "system": system, "type": "notation"}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = "test_mapping.db"
config = deepcopy(das_readconfig())
dburi = config["mongodb"]["dburi"]
logger = PrintManager("TestDASMapping", verbose=debug)
config["logger"] = logger
config["verbose"] = debug
dbname = "test_mapping"
collname = "db"
config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {
"presentation": {
"block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]
},
"type": "presentation",
}
self.coll.insert(self.pmap)
ver_token = verification_token(self.coll.find(exhaust=True))
rec = {"verification_token": ver_token, "type": "verification_token"}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
self.mgr.init()
apiversion = "DBS_2_0_8"
url = "http://a.com"
dformat = "JSON"
expire = 100
api = "listRuns"
params = {"apiversion": apiversion, "path": "required", "api": api}
rec = {
"system": "dbs",
"urn": api,
"format": dformat,
"url": url,
"params": params,
"expire": expire,
"lookup": "run",
"wild_card": "*",
"das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")],
"type": "service",
}
self.mgr.add(rec)
smap = {
api: {
"url": url,
"expire": expire,
"keys": ["run"],
"format": dformat,
"wild_card": "*",
"cert": None,
"ckey": None,
"services": "",
"lookup": "run",
"params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"},
}
}
rec = {
"system": "dbs",
"urn": "listBlocks",
"format": dformat,
"url": url,
"expire": expire,
"lookup": "block",
"params": {
"apiversion": apiversion,
"api": "listBlocks",
"block_name": "*",
"storage_element_name": "*",
"user_type": "NORMAL",
},
"das_map": [
{"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"},
{
"das_key": "site",
"rec_key": "site.se",
"api_arg": "storage_element_name",
"pattern": "re.compile('([a-zA-Z0-9]+\.){2}')",
},
],
"type": "service",
}
self.mgr.add(rec)
system = "dbs"
api = "listBlocks"
daskey = "block"
rec_key = "block.name"
api_input = "block_name"
res = self.mgr.list_systems()
self.assertEqual(["dbs"], res)
res = self.mgr.list_apis()
# self.assertEqual([api], res)
res.sort()
self.assertEqual(["listBlocks", "listRuns"], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ""
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {
"system": system,
"type": "notation",
"notations": [
{"api_output": "storage_element_name", "rec_key": "se", "api": ""},
{"api_output": "number_of_events", "rec_key": "nevents", "api": ""},
],
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, "number_of_events")
self.assertEqual("nevents", res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(["block", "site"], res)
# build service map
smap.update(
{
api: {
"url": url,
"expire": expire,
"cert": None,
"ckey": None,
"keys": ["block", "site"],
"format": dformat,
"wild_card": "*",
"services": "",
"lookup": daskey,
"params": {
"storage_element_name": "*",
"api": api,
"block_name": "*",
"user_type": "NORMAL",
"apiversion": "DBS_2_0_8",
},
}
}
)
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
self.mgr.init()
expect = self.pmap["presentation"]["block"]
result = self.mgr.presentation("block")
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
self.mgr.init()
system = "test"
rec = {
"notations": [
{"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""},
],
"system": system,
"type": "notation",
}
self.mgr.add(rec)
expect = rec["notations"]
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class QLManager(object):
"""
DAS QL manager.
"""
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def parse(self, query):
"""
Parse input query and return query in MongoDB form.
"""
mongo_query = self.mongo_query(query)
self.convert2skeys(mongo_query)
return mongo_query
def mongo_query(self, query):
"""
Return mongo query for provided input query
"""
mongo_query = parse_query(query, self.daskeys, self.dasservices,
self.verbose)
if set(mongo_query.keys()) & set(['fields', 'spec']) != \
set(['fields', 'spec']):
raise Exception('Invalid MongoDB query %s' % mongo_query)
if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1:
raise Exception(ambiguous_msg(query, mongo_query['spec'].keys()))
for key, val in mongo_query['spec'].items():
if isinstance(val, list):
raise Exception(ambiguos_val_msg(query, key, val))
return mongo_query
def convert2skeys(self, mongo_query):
"""
Convert DAS input keys into DAS selection keys.
"""
if not mongo_query['spec']:
for key in mongo_query['fields']:
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key)
if mapkey:
mongo_query['spec'][mapkey] = '*'
return
spec = mongo_query['spec']
to_replace = []
for key, val in spec.items():
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key, val)
if mapkey and mapkey != key and \
key in mongo_query['spec']:
to_replace.append((key, mapkey))
continue
for key, mapkey in to_replace:
if key in mongo_query['spec']:
mongo_query['spec'][mapkey] = mongo_query['spec'][key]
del mongo_query['spec'][key]
def services(self, query):
"""Find out DAS services to use for provided query"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
slist = []
# look-up services from Mapping DB
for key in skeys + [i for i in cond.keys()]:
for service, keys in self.daskeysmap.items():
if service not in self.dasservices:
continue
value = cond.get(key, None)
daskeys = self.dasmapping.find_daskey(service, key, value)
if set(keys) & set(daskeys) and service not in slist:
slist.append(service)
# look-up special key condition
requested_system = query.get('system', None)
if requested_system:
if isinstance(requested_system, basestring):
requested_system = [requested_system]
return list(set(slist) & set(requested_system))
return slist
def service_apis_map(self, query):
"""
Find out which APIs correspond to provided query.
Return a map of found services and their apis.
"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
adict = {}
mapkeys = [
key for key in list(cond.keys()) if key not in das_special_keys()
]
services = self.services(query)
for srv in services:
alist = self.dasmapping.list_apis(srv)
for api in alist:
daskeys = self.dasmapping.api_info(srv, api)['das_map']
maps = [r['rec_key'] for r in daskeys]
if set(mapkeys) & set(maps) == set(mapkeys):
if srv in adict:
new_list = adict[srv] + [api]
adict[srv] = list(set(new_list))
else:
adict[srv] = [api]
return adict
def params(self, query):
"""
Return dictionary of parameters to be used in DAS Core:
selection keys, conditions and services.
"""
skeys, cond = decompose(query)
services = []
for srv in self.services(query):
if srv not in services:
services.append(srv)
return dict(selkeys=skeys, conditions=cond, services=services)
class QLManager(object):
"""
DAS QL manager.
"""
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
def parse(self, query):
"""
Parse input query and return query in MongoDB form.
"""
mongo_query = self.mongo_query(query)
self.convert2skeys(mongo_query)
return mongo_query
def mongo_query(self, query):
"""
Return mongo query for provided input query
"""
mongo_query = parse_query(query, self.daskeys, self.dasservices, self.verbose)
if set(mongo_query.keys()) & set(['fields', 'spec']) != \
set(['fields', 'spec']):
raise Exception('Invalid MongoDB query %s' % mongo_query)
if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1:
raise Exception(ambiguous_msg(query, mongo_query['spec'].keys()))
for key, val in mongo_query['spec'].items():
if isinstance(val, list):
raise Exception(ambiguos_val_msg(query, key, val))
return mongo_query
def convert2skeys(self, mongo_query):
"""
Convert DAS input keys into DAS selection keys.
"""
if not mongo_query['spec']:
for key in mongo_query['fields']:
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key)
if mapkey:
mongo_query['spec'][mapkey] = '*'
return
spec = mongo_query['spec']
to_replace = []
for key, val in spec.items():
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key, val)
if mapkey and mapkey != key and \
key in mongo_query['spec']:
to_replace.append((key, mapkey))
continue
for key, mapkey in to_replace:
if key in mongo_query['spec']:
mongo_query['spec'][mapkey] = mongo_query['spec'][key]
del mongo_query['spec'][key]
def services(self, query):
"""Find out DAS services to use for provided query"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
slist = []
# look-up services from Mapping DB
for key in skeys + [i for i in cond.keys()]:
for service, keys in self.daskeysmap.items():
if service not in self.dasservices:
continue
value = cond.get(key, None)
daskeys = self.dasmapping.find_daskey(service, key, value)
if set(keys) & set(daskeys) and service not in slist:
slist.append(service)
# look-up special key condition
requested_system = query.get('system', None)
if requested_system:
if isinstance(requested_system, basestring):
requested_system = [requested_system]
return list( set(slist) & set(requested_system) )
return slist
def service_apis_map(self, query):
"""
Find out which APIs correspond to provided query.
Return a map of found services and their apis.
"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
adict = {}
mapkeys = [key for key in cond.keys() if key not in das_special_keys()]
services = self.services(query)
for srv in services:
alist = self.dasmapping.list_apis(srv)
for api in alist:
daskeys = self.dasmapping.api_info(srv, api)['das_map']
maps = [r['rec_key'] for r in daskeys]
if set(mapkeys) & set(maps) == set(mapkeys):
if srv in adict:
new_list = adict[srv] + [api]
adict[srv] = list( set(new_list) )
else:
adict[srv] = [api]
return adict
def params(self, query):
"""
Return dictionary of parameters to be used in DAS Core:
selection keys, conditions and services.
"""
skeys, cond = decompose(query)
services = []
for srv in self.services(query):
if srv not in services:
services.append(srv)
return dict(selkeys=skeys, conditions=cond, services=services)
def main():
"Main function"
optmgr = DASOptionParser()
(opts, _args) = optmgr.getOpt()
dburi = 'mongodb://%s:%s' % (opts.host, opts.port)
dasconfig = das_readconfig()
dbname, colname = opts.db.split('.')
mongodb = dict(dburi=dburi)
mappingdb = dict(dbname=dbname, collname=colname)
config = dict(verbose=opts.debug, mappingdb=mappingdb,
mongodb=mongodb, services=dasconfig['das'].get('services', []))
mgr = DASMapping(config)
if opts.listapis:
apis = mgr.list_apis(opts.system)
print apis
sys.exit(0)
if opts.listkeys:
keys = mgr.daskeys(opts.system)
print keys
sys.exit(0)
if opts.umap:
for rec in read_service_map(opts.umap, field='uri'):
if opts.debug:
print rec
spec = {'url':rec['url'], 'urn':rec['urn']}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.nmap:
for rec in read_service_map(opts.nmap, field='notations'):
if opts.debug:
print rec
system = rec['system']
spec = {'notations':{'$exists':True}, 'system':system}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.pmap:
for rec in read_service_map(opts.pmap, field='presentation'):
if opts.debug:
print rec
spec = {'presentation':{'$exists':True}}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.clean:
mgr.delete_db()
mgr.create_db()
# I need to clear DAS cache/merge since I don't know
# a-priory what kind of changes new maps will bring
conn = db_connection(dburi)
dbname = dasconfig['dasdb']['dbname']
cache = conn[dbname][dasconfig['dasdb']['cachecollection']]
cache.remove({})
merge = conn[dbname][dasconfig['dasdb']['mergecollection']]
merge.remove({})
if opts.remove:
mgr.remove(opts.remove)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = Connection(dburi)
conn.drop_database(dbname)
coll = conn[dbname][collname]
self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
{"ui": "Block size", "das": "block.size"}]}}
coll.insert(self.pmap)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
self.mgr.create_db()
apiversion = 'DBS_2_0_8'
url = 'http://a.com'
dformat = 'JSON'
expire = 100
api = 'listRuns'
params = { 'apiversion':apiversion, 'path' : 'required', 'api':api}
rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url,
'params': params, 'expire':expire, "wild_card": "*",
'daskeys' : [dict(key='run', map='run.run_number', pattern='')],
'das2api' : [
dict(api_param='path', das_key='dataset', pattern=""),
]
}
self.mgr.add(rec)
res = self.mgr.check_dasmap('dbs', api, 'run.bfield')
self.assertEqual(False, res)
res = self.mgr.check_dasmap('dbs', api, 'run.run_number')
self.assertEqual(True, res)
smap = {api: {'url':url, 'expire':expire, 'keys': ['run'],
'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None,
'params': {'path': 'required', 'api': api,
'apiversion': 'DBS_2_0_8'}
}
}
rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat,
'url':url, 'expire': expire,
'params' : {'apiversion': apiversion, 'api': 'listBlocks',
'block_name':'*', 'storage_element_name':'*',
'user_type':'NORMAL'},
'daskeys': [
{'key':'block', 'map':'block.name', 'pattern':''},
],
'das2api': [
{'api_param':'storage_element_name',
'das_key':'site',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'storage_element_name',
'das_key':'site.se',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'block_name',
'das_key':'block',
'pattern':""},
{'api_param':'block_name',
'das_key':'block.name',
'pattern':""},
]
}
self.mgr.add(rec)
system = 'dbs'
api = 'listBlocks'
daskey = 'block'
primkey = 'block.name'
api_input = 'block_name'
res = self.mgr.list_systems()
self.assertEqual(['dbs'], res)
res = self.mgr.list_apis()
# self.assertEqual([api], res)
res.sort()
self.assertEqual(['listBlocks', 'listRuns'], res)
res = self.mgr.lookup_keys(system, daskey)
self.assertEqual([primkey], res)
value = ''
res = self.mgr.das2api(system, daskey, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, daskey, value, api)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey, primkey], res)
# adding notations
notations = {'system':system,
'notations':[
{'notation':'storage_element_name', 'map':'se', 'api':''},
{'notation':'number_of_events', 'map':'nevents', 'api':''},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual([daskey], res)
# build service map
smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
'keys': ['block'], 'format':dformat, "wild_card": "*",
'params': {'storage_element_name': '*', 'api':api,
'block_name': '*', 'user_type': 'NORMAL',
'apiversion': 'DBS_2_0_8'}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
self.mgr.create_db()
# rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}}
# self.mgr.add(rec)
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
self.mgr.create_db()
system = "test"
rec = {'notations': [
{"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""},
{"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""},
{"notation": "admin.contacts.cms_name", "map":"site.name", "api":""}
], "system": system}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class QLManager(object):
"""
DAS QL manager.
"""
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.analytics = DASAnalytics(config)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
parserdir = config['das']['parserdir']
self.parserdir = parserdir
self.enabledb = config['parserdb']['enable']
if self.enabledb:
self.parserdb = DASParserDB(config)
def parse(self, query):
"""
Parse input query and return query in MongoDB form.
Optionally parsed query can be written into analytics DB.
"""
mongo_query = self.mongo_query(query)
self.convert2skeys(mongo_query)
return mongo_query
def add_to_analytics(self, query, mongo_query):
"Add DAS query to analytics DB"
self.analytics.add_query(query, mongo_query)
def get_ply_query(self, query):
"""
Get ply object for given query. Since we rely on PLY package and it may
fail under the load we use couple of trials.
"""
ply_query = ply_parse_query(query, self.daskeys, self.dasservices,
self.parserdir, self.verbose)
return ply_query
def mongo_query(self, query):
"""
Return mongo query for provided input query
"""
mongo_query = None
if self.verbose:
ply_output(query, self.daskeys, self.dasservices,
self.parserdir, self.verbose)
parse_again = True
if self.enabledb:
status, value = self.parserdb.lookup_query(query)
if status == PARSERCACHE_VALID and \
len(last_key_pattern.findall(query)) == 0:
mongo_query = value
parse_again = False
elif status == PARSERCACHE_INVALID:
# we unable to find query in parserdb, so will parse again
parse_again = True
else:
ply_query = self.get_ply_query(query)
if ply_query:
try:
mongo_query = ply2mongo(ply_query)
parse_again = False
except Exception as exc:
msg = "Fail in ply2mongo, query=%s, ply_query=%s" \
% (query, ply_query)
print msg
try:
self.parserdb.insert_valid_query(query, mongo_query)
except Exception as exc:
msg = "Fail to insert into parserdb, exception=%s" \
% str(exc)
print_exc(msg, print_traceback=True)
if parse_again:
try:
ply_query = self.get_ply_query(query)
mongo_query = ply2mongo(ply_query)
except Exception as exc:
msg = "Fail to parse query='%s'" % query
print_exc(msg, print_traceback=False)
raise exc
if set(mongo_query.keys()) & set(['fields', 'spec']) != \
set(['fields', 'spec']):
raise Exception('Invalid MongoDB query %s' % mongo_query)
if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1:
raise Exception(ambiguous_msg(query, mongo_query['spec'].keys()))
for key, val in mongo_query['spec'].iteritems():
if isinstance(val, list):
raise Exception(ambiguos_val_msg(query, key, val))
return mongo_query
def convert2skeys(self, mongo_query):
"""
Convert DAS input keys into DAS selection keys.
"""
if not mongo_query['spec']:
for key in mongo_query['fields']:
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key)
if mapkey:
mongo_query['spec'][mapkey] = '*'
return
spec = mongo_query['spec']
to_replace = []
for key, val in spec.iteritems():
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key, val)
if mapkey and mapkey != key and \
key in mongo_query['spec']:
to_replace.append((key, mapkey))
continue
for key, mapkey in to_replace:
if key in mongo_query['spec']:
mongo_query['spec'][mapkey] = mongo_query['spec'][key]
del mongo_query['spec'][key]
def services(self, query):
"""Find out DAS services to use for provided query"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
slist = []
# look-up services from Mapping DB
for key in skeys + [i for i in cond.keys()]:
for service, keys in self.daskeysmap.iteritems():
if service not in self.dasservices:
continue
value = cond.get(key, None)
daskeys = self.dasmapping.find_daskey(service, key, value)
if set(keys) & set(daskeys) and service not in slist:
slist.append(service)
# look-up special key condition
requested_system = query.get('system', None)
if requested_system:
if isinstance(requested_system, basestring):
requested_system = [requested_system]
return list( set(slist) & set(requested_system) )
return slist
def service_apis_map(self, query):
"""
Find out which APIs correspond to provided query.
Return a map of found services and their apis.
"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
adict = {}
mapkeys = [key for key in cond.keys() if key not in das_special_keys()]
services = self.services(query)
for srv in services:
alist = self.dasmapping.list_apis(srv)
for api in alist:
daskeys = self.dasmapping.api_info(srv, api)['das_map']
maps = [r['rec_key'] for r in daskeys]
if set(mapkeys) & set(maps) == set(mapkeys):
if srv in adict:
new_list = adict[srv] + [api]
adict[srv] = list( set(new_list) )
else:
adict[srv] = [api]
return adict
def params(self, query):
"""
Return dictionary of parameters to be used in DAS Core:
selection keys, conditions and services.
"""
skeys, cond = decompose(query)
services = []
for srv in self.services(query):
if srv not in services:
services.append(srv)
return dict(selkeys=skeys, conditions=cond, services=services)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi,
dbname=dbname,
collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {
"presentation": {
"block": [{
"ui": "Block name",
"das": "block.name"
}, {
"ui": "Block size",
"das": "block.size"
}]
},
"type": "presentation"
}
self.coll.insert(self.pmap)
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
dformat = 'JSON'
system = 'dbs3'
expire = 100
rec = {
'system':
system,
'urn':
'acquisitioneras',
'format':
dformat,
'instances': ['prod/global'],
'url':
url,
'expire':
expire,
'lookup':
'era',
'params': {},
'das_map': [{
"das_key": "era",
"rec_key": "era.name",
"api_arg": "era"
}],
'type':
'service'
}
self.coll.insert(rec)
ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
rec = {'verification_token': ver_token, 'type': 'verification_token'}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
system = 'dbs3'
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
dformat = 'JSON'
expire = 100
instances = ["prod/global", "prod/phys01"]
api = 'primarydatasets'
params = {"primary_ds_name": "*"}
rec = {
'system':
system,
'urn':
api,
'format':
dformat,
'url':
url,
'instances':
instances,
'params':
params,
'expire':
expire,
'lookup':
'primary_dataset',
'wild_card':
'*',
'das_map': [
dict(das_key='primary_dataset',
rec_key='primary_dataset.name',
api_arg='primary_dataset')
],
'type':
'service'
}
self.mgr.add(rec)
smap = {
api: {
'url': url,
'expire': expire,
'keys': ['primary_dataset'],
'format': dformat,
'wild_card': '*',
'cert': None,
'ckey': None,
'services': '',
'lookup': 'primary_dataset',
'params': params
}
}
rec = {
'system':
system,
'urn':
'datasetaccesstypes',
'format':
dformat,
'instances':
instances,
'url':
url,
'expire':
expire,
'lookup':
'status',
'params': {
'status': '*'
},
'das_map': [{
"das_key": "status",
"rec_key": "status.name",
"api_arg": "status"
}],
'type':
'service'
}
self.mgr.add(rec)
api = 'datasetaccesstypes'
daskey = 'status'
rec_key = 'status.name'
api_input = 'status'
res = self.mgr.list_systems()
self.assertEqual([system], res)
res = self.mgr.list_apis()
res.sort()
self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ''
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {
'system':
system,
'type':
'notation',
'notations': [
{
'api_output': 'storage_element_name',
'rec_key': 'se',
'api': ''
},
{
'api_output': 'number_of_events',
'rec_key': 'nevents',
'api': ''
},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(['status'], res)
# build service map
smap.update({
api: {
'url': url,
'expire': expire,
'cert': None,
'ckey': None,
'keys': ['status'],
'format': dformat,
'wild_card': '*',
'services': '',
'lookup': daskey,
'params': {
"status": "*"
}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
system = "test"
rec = {
'notations': [{
"api_output": "site.resource_element.cms_name",
"rec_key": "site.name",
"api": ""
}, {
"api_output": "site.resource_pledge.cms_name",
"rec_key": "site.name",
"api": ""
}, {
"api_output": "admin.contacts.cms_name",
"rec_key": "site.name",
"api": ""
}],
"system":
system,
"type":
"notation"
}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
