def init(self): """Takes care of MongoDB connection""" try: indexes = [('dataset', DESCENDING), ('site', DESCENDING), ('ts', DESCENDING)] for index in indexes: create_indexes(self.coll, [index]) dasmapping = DASMapping(self.dasconfig) service_name = self.config.get('name', 'combined') service_api = self.config.get('api', 'dataset4site_release') mapping = dasmapping.servicemap(service_name) self.urls = mapping[service_api]['services'] self.expire = mapping[service_api]['expire'] services = self.dasconfig['services'] if not self.wthr: # Worker thread which update dbs/phedex DB self.wthr = start_new_thread('dbs_phedex_worker', worker, \ (self.urls, which_dbs, self.uri, \ self.dbname, self.collname, self.expire)) msg = "### DBSPhedexService:init started" print(msg) except Exception as exc: print("### Fail DBSPhedexService:init\n", str(exc)) self.urls = None self.expire = 60 self.wthr = None
def init(self): "Takes care of MongoDB connection since DASMapping requires it" try: dasmapping = DASMapping(self.dasconfig) mapping = dasmapping.servicemap(self.service_name) self.urls = mapping[self.service_api]['services'] self.expire = mapping[self.service_api]['expire'] except Exception as _exp: pass
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]}, "type": "presentation"} self.coll.insert(self.pmap) url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/' dformat = 'JSON' system = 'dbs3' expire = 100 rec = {'system':system, 'urn': 'acquisitioneras', 'format':dformat, 'instances': ['prod/global'], 'url':url, 'expire': expire, 'lookup': 'era', 'params' : {}, 'das_map': [ {"das_key": "era", "rec_key":"era.name", "api_arg":"era"} ], 'type': 'service' } self.coll.insert(rec) ver_token = verification_token(self.coll.find(**PYMONGO_OPTS)) rec = {'verification_token':ver_token, 'type':'verification_token'} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() system = 'dbs3' url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' dformat = 'JSON' expire = 100 instances = ["prod/global", "prod/phys01"] api = 'primarydatasets' params = {"primary_ds_name":"*"} rec = {'system':system, 'urn':api, 'format':dformat, 'url':url, 'instances': instances, 'params': params, 'expire':expire, 'lookup': 'primary_dataset', 'wild_card':'*', 'das_map' : [dict(das_key='primary_dataset', rec_key='primary_dataset.name', api_arg='primary_dataset')], 'type': 'service' } self.mgr.add(rec) smap = {api: {'url':url, 'expire':expire, 'keys': ['primary_dataset'], 'format': dformat, 'wild_card':'*', 'cert':None, 'ckey': None, 'services': '', 'lookup': 'primary_dataset', 'params': params } } rec = {'system':system, 'urn': 'datasetaccesstypes', 'format':dformat, 'instances': instances, 'url':url, 'expire': expire, 'lookup': 'status', 'params' : {'status':'*'}, 'das_map': [ {"das_key": "status", "rec_key":"status.name", "api_arg":"status"} ], 'type': 'service' } self.mgr.add(rec) api = 'datasetaccesstypes' daskey = 'status' rec_key = 'status.name' api_input = 'status' res = self.mgr.list_systems() self.assertEqual([system], res) res = self.mgr.list_apis() res.sort() self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = '' res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = {'system':system, 'type': 'notation', 'notations':[ {'api_output':'storage_element_name', 'rec_key':'se', 'api':''}, {'api_output':'number_of_events', 'rec_key':'nevents', 'api':''}, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(['status'], res) # build service map smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None, 'keys': ['status'], 'format':dformat, 'wild_card':'*', 'services': '', 'lookup': daskey, 'params': {"status": "*"} } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" system = "test" rec = {'notations': [ {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "admin.contacts.cms_name", "rec_key":"site.name", "api":""} ], "system": system, "type": "notation"} self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = "test_mapping.db" config = deepcopy(das_readconfig()) dburi = config["mongodb"]["dburi"] logger = PrintManager("TestDASMapping", verbose=debug) config["logger"] = logger config["verbose"] = debug dbname = "test_mapping" collname = "db" config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = { "presentation": { "block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}] }, "type": "presentation", } self.coll.insert(self.pmap) ver_token = verification_token(self.coll.find(exhaust=True)) rec = {"verification_token": ver_token, "type": "verification_token"} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() self.mgr.init() apiversion = "DBS_2_0_8" url = "http://a.com" dformat = "JSON" expire = 100 api = "listRuns" params = {"apiversion": apiversion, "path": "required", "api": api} rec = { "system": "dbs", "urn": api, "format": dformat, "url": url, "params": params, "expire": expire, "lookup": "run", "wild_card": "*", "das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")], "type": "service", } self.mgr.add(rec) smap = { api: { "url": url, "expire": expire, "keys": ["run"], "format": dformat, "wild_card": "*", "cert": None, "ckey": None, "services": "", "lookup": "run", "params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"}, } } rec = { "system": "dbs", "urn": "listBlocks", "format": dformat, "url": url, "expire": expire, "lookup": "block", "params": { "apiversion": apiversion, "api": "listBlocks", "block_name": "*", "storage_element_name": "*", "user_type": "NORMAL", }, "das_map": [ {"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"}, { "das_key": "site", "rec_key": "site.se", "api_arg": "storage_element_name", "pattern": "re.compile('([a-zA-Z0-9]+\.){2}')", }, ], "type": "service", } self.mgr.add(rec) system = "dbs" api = "listBlocks" daskey = "block" rec_key = "block.name" api_input = "block_name" res = self.mgr.list_systems() self.assertEqual(["dbs"], res) res = self.mgr.list_apis() # self.assertEqual([api], res) res.sort() self.assertEqual(["listBlocks", "listRuns"], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = "" res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = { "system": system, "type": "notation", "notations": [ {"api_output": "storage_element_name", "rec_key": "se", "api": ""}, {"api_output": "number_of_events", "rec_key": "nevents", "api": ""}, ], } self.mgr.add(notations) res = self.mgr.notation2das(system, "number_of_events") self.assertEqual("nevents", res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(["block", "site"], res) # build service map smap.update( { api: { "url": url, "expire": expire, "cert": None, "ckey": None, "keys": ["block", "site"], "format": dformat, "wild_card": "*", "services": "", "lookup": daskey, "params": { "storage_element_name": "*", "api": api, "block_name": "*", "user_type": "NORMAL", "apiversion": "DBS_2_0_8", }, } } ) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" self.mgr.init() expect = self.pmap["presentation"]["block"] result = self.mgr.presentation("block") self.assertEqual(expect, result) def test_notations(self): """test notations method""" self.mgr.init() system = "test" rec = { "notations": [ {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""}, ], "system": system, "type": "notation", } self.mgr.add(rec) expect = rec["notations"] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = Connection(dburi) conn.drop_database(dbname) coll = conn[dbname][collname] self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]}} coll.insert(self.pmap) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() self.mgr.create_db() apiversion = 'DBS_2_0_8' url = 'http://a.com' dformat = 'JSON' expire = 100 api = 'listRuns' params = { 'apiversion':apiversion, 'path' : 'required', 'api':api} rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url, 'params': params, 'expire':expire, "wild_card": "*", 'daskeys' : [dict(key='run', map='run.run_number', pattern='')], 'das2api' : [ dict(api_param='path', das_key='dataset', pattern=""), ] } self.mgr.add(rec) res = self.mgr.check_dasmap('dbs', api, 'run.bfield') self.assertEqual(False, res) res = self.mgr.check_dasmap('dbs', api, 'run.run_number') self.assertEqual(True, res) smap = {api: {'url':url, 'expire':expire, 'keys': ['run'], 'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None, 'params': {'path': 'required', 'api': api, 'apiversion': 'DBS_2_0_8'} } } rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat, 'url':url, 'expire': expire, 'params' : {'apiversion': apiversion, 'api': 'listBlocks', 'block_name':'*', 'storage_element_name':'*', 'user_type':'NORMAL'}, 'daskeys': [ {'key':'block', 'map':'block.name', 'pattern':''}, ], 'das2api': [ {'api_param':'storage_element_name', 'das_key':'site', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'storage_element_name', 'das_key':'site.se', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'block_name', 'das_key':'block', 'pattern':""}, {'api_param':'block_name', 'das_key':'block.name', 'pattern':""}, ] } self.mgr.add(rec) system = 'dbs' api = 'listBlocks' daskey = 'block' primkey = 'block.name' api_input = 'block_name' res = self.mgr.list_systems() self.assertEqual(['dbs'], res) res = self.mgr.list_apis() # self.assertEqual([api], res) res.sort() self.assertEqual(['listBlocks', 'listRuns'], res) res = self.mgr.lookup_keys(system, daskey) self.assertEqual([primkey], res) value = '' res = self.mgr.das2api(system, daskey, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, daskey, value, api) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey, primkey], res) # adding notations notations = {'system':system, 'notations':[ {'notation':'storage_element_name', 'map':'se', 'api':''}, {'notation':'number_of_events', 'map':'nevents', 'api':''}, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual([daskey], res) # build service map smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None, 'keys': ['block'], 'format':dformat, "wild_card": "*", 'params': {'storage_element_name': '*', 'api':api, 'block_name': '*', 'user_type': 'NORMAL', 'apiversion': 'DBS_2_0_8'} } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" self.mgr.create_db() # rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}} # self.mgr.add(rec) expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" self.mgr.create_db() system = "test" rec = {'notations': [ {"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""}, {"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""}, {"notation": "admin.contacts.cms_name", "map":"site.name", "api":""} ], "system": system} self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = { "presentation": { "block": [{ "ui": "Block name", "das": "block.name" }, { "ui": "Block size", "das": "block.size" }] }, "type": "presentation" } self.coll.insert(self.pmap) url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/' dformat = 'JSON' system = 'dbs3' expire = 100 rec = { 'system': system, 'urn': 'acquisitioneras', 'format': dformat, 'instances': ['prod/global'], 'url': url, 'expire': expire, 'lookup': 'era', 'params': {}, 'das_map': [{ "das_key": "era", "rec_key": "era.name", "api_arg": "era" }], 'type': 'service' } self.coll.insert(rec) ver_token = verification_token(self.coll.find(**PYMONGO_OPTS)) rec = {'verification_token': ver_token, 'type': 'verification_token'} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() system = 'dbs3' url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' dformat = 'JSON' expire = 100 instances = ["prod/global", "prod/phys01"] api = 'primarydatasets' params = {"primary_ds_name": "*"} rec = { 'system': system, 'urn': api, 'format': dformat, 'url': url, 'instances': instances, 'params': params, 'expire': expire, 'lookup': 'primary_dataset', 'wild_card': '*', 'das_map': [ dict(das_key='primary_dataset', rec_key='primary_dataset.name', api_arg='primary_dataset') ], 'type': 'service' } self.mgr.add(rec) smap = { api: { 'url': url, 'expire': expire, 'keys': ['primary_dataset'], 'format': dformat, 'wild_card': '*', 'cert': None, 'ckey': None, 'services': '', 'lookup': 'primary_dataset', 'params': params } } rec = { 'system': system, 'urn': 'datasetaccesstypes', 'format': dformat, 'instances': instances, 'url': url, 'expire': expire, 'lookup': 'status', 'params': { 'status': '*' }, 'das_map': [{ "das_key": "status", "rec_key": "status.name", "api_arg": "status" }], 'type': 'service' } self.mgr.add(rec) api = 'datasetaccesstypes' daskey = 'status' rec_key = 'status.name' api_input = 'status' res = self.mgr.list_systems() self.assertEqual([system], res) res = self.mgr.list_apis() res.sort() self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = '' res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = { 'system': system, 'type': 'notation', 'notations': [ { 'api_output': 'storage_element_name', 'rec_key': 'se', 'api': '' }, { 'api_output': 'number_of_events', 'rec_key': 'nevents', 'api': '' }, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(['status'], res) # build service map smap.update({ api: { 'url': url, 'expire': expire, 'cert': None, 'ckey': None, 'keys': ['status'], 'format': dformat, 'wild_card': '*', 'services': '', 'lookup': daskey, 'params': { "status": "*" } } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" system = "test" rec = { 'notations': [{ "api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": "" }, { "api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": "" }, { "api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": "" }], "system": system, "type": "notation" } self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)