def init(self):
"""Takes care of MongoDB connection"""
try:
indexes = [('dataset', DESCENDING), ('site', DESCENDING),
('ts', DESCENDING)]
for index in indexes:
create_indexes(self.coll, [index])
dasmapping = DASMapping(self.dasconfig)
service_name = self.config.get('name', 'combined')
service_api = self.config.get('api', 'dataset4site_release')
mapping = dasmapping.servicemap(service_name)
self.urls = mapping[service_api]['services']
self.expire = mapping[service_api]['expire']
services = self.dasconfig['services']
if not self.wthr:
# Worker thread which update dbs/phedex DB
self.wthr = start_new_thread('dbs_phedex_worker', worker, \
(self.urls, which_dbs, self.uri, \
self.dbname, self.collname, self.expire))
msg = "### DBSPhedexService:init started"
print(msg)
except Exception as exc:
print("### Fail DBSPhedexService:init\n", str(exc))
self.urls = None
self.expire = 60
self.wthr = None
def init(self):
"Takes care of MongoDB connection since DASMapping requires it"
try:
dasmapping = DASMapping(self.dasconfig)
mapping = dasmapping.servicemap(self.service_name)
self.urls = mapping[self.service_api]['services']
self.expire = mapping[self.service_api]['expire']
except Exception as _exp:
pass
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
{"ui": "Block size", "das": "block.size"}]}, "type": "presentation"}
self.coll.insert(self.pmap)
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
dformat = 'JSON'
system = 'dbs3'
expire = 100
rec = {'system':system, 'urn': 'acquisitioneras', 'format':dformat,
'instances': ['prod/global'],
'url':url, 'expire': expire, 'lookup': 'era',
'params' : {},
'das_map': [
{"das_key": "era", "rec_key":"era.name", "api_arg":"era"}
],
'type': 'service'
}
self.coll.insert(rec)
ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
rec = {'verification_token':ver_token, 'type':'verification_token'}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
system = 'dbs3'
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
dformat = 'JSON'
expire = 100
instances = ["prod/global", "prod/phys01"]
api = 'primarydatasets'
params = {"primary_ds_name":"*"}
rec = {'system':system, 'urn':api, 'format':dformat, 'url':url,
'instances': instances,
'params': params, 'expire':expire, 'lookup': 'primary_dataset', 'wild_card':'*',
'das_map' : [dict(das_key='primary_dataset',
rec_key='primary_dataset.name',
api_arg='primary_dataset')],
'type': 'service'
}
self.mgr.add(rec)
smap = {api: {'url':url, 'expire':expire, 'keys': ['primary_dataset'],
'format': dformat, 'wild_card':'*', 'cert':None, 'ckey': None,
'services': '', 'lookup': 'primary_dataset',
'params': params }
}
rec = {'system':system, 'urn': 'datasetaccesstypes', 'format':dformat,
'instances': instances,
'url':url, 'expire': expire, 'lookup': 'status',
'params' : {'status':'*'},
'das_map': [
{"das_key": "status", "rec_key":"status.name", "api_arg":"status"}
],
'type': 'service'
}
self.mgr.add(rec)
api = 'datasetaccesstypes'
daskey = 'status'
rec_key = 'status.name'
api_input = 'status'
res = self.mgr.list_systems()
self.assertEqual([system], res)
res = self.mgr.list_apis()
res.sort()
self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ''
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {'system':system, 'type': 'notation',
'notations':[
{'api_output':'storage_element_name', 'rec_key':'se', 'api':''},
{'api_output':'number_of_events', 'rec_key':'nevents', 'api':''},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(['status'], res)
# build service map
smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
'keys': ['status'], 'format':dformat, 'wild_card':'*',
'services': '', 'lookup': daskey,
'params': {"status": "*"}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
system = "test"
rec = {'notations': [
{"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "admin.contacts.cms_name", "rec_key":"site.name", "api":""}
], "system": system, "type": "notation"}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = "test_mapping.db"
config = deepcopy(das_readconfig())
dburi = config["mongodb"]["dburi"]
logger = PrintManager("TestDASMapping", verbose=debug)
config["logger"] = logger
config["verbose"] = debug
dbname = "test_mapping"
collname = "db"
config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {
"presentation": {
"block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]
},
"type": "presentation",
}
self.coll.insert(self.pmap)
ver_token = verification_token(self.coll.find(exhaust=True))
rec = {"verification_token": ver_token, "type": "verification_token"}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
self.mgr.init()
apiversion = "DBS_2_0_8"
url = "http://a.com"
dformat = "JSON"
expire = 100
api = "listRuns"
params = {"apiversion": apiversion, "path": "required", "api": api}
rec = {
"system": "dbs",
"urn": api,
"format": dformat,
"url": url,
"params": params,
"expire": expire,
"lookup": "run",
"wild_card": "*",
"das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")],
"type": "service",
}
self.mgr.add(rec)
smap = {
api: {
"url": url,
"expire": expire,
"keys": ["run"],
"format": dformat,
"wild_card": "*",
"cert": None,
"ckey": None,
"services": "",
"lookup": "run",
"params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"},
}
}
rec = {
"system": "dbs",
"urn": "listBlocks",
"format": dformat,
"url": url,
"expire": expire,
"lookup": "block",
"params": {
"apiversion": apiversion,
"api": "listBlocks",
"block_name": "*",
"storage_element_name": "*",
"user_type": "NORMAL",
},
"das_map": [
{"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"},
{
"das_key": "site",
"rec_key": "site.se",
"api_arg": "storage_element_name",
"pattern": "re.compile('([a-zA-Z0-9]+\.){2}')",
},
],
"type": "service",
}
self.mgr.add(rec)
system = "dbs"
api = "listBlocks"
daskey = "block"
rec_key = "block.name"
api_input = "block_name"
res = self.mgr.list_systems()
self.assertEqual(["dbs"], res)
res = self.mgr.list_apis()
# self.assertEqual([api], res)
res.sort()
self.assertEqual(["listBlocks", "listRuns"], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ""
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {
"system": system,
"type": "notation",
"notations": [
{"api_output": "storage_element_name", "rec_key": "se", "api": ""},
{"api_output": "number_of_events", "rec_key": "nevents", "api": ""},
],
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, "number_of_events")
self.assertEqual("nevents", res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(["block", "site"], res)
# build service map
smap.update(
{
api: {
"url": url,
"expire": expire,
"cert": None,
"ckey": None,
"keys": ["block", "site"],
"format": dformat,
"wild_card": "*",
"services": "",
"lookup": daskey,
"params": {
"storage_element_name": "*",
"api": api,
"block_name": "*",
"user_type": "NORMAL",
"apiversion": "DBS_2_0_8",
},
}
}
)
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
self.mgr.init()
expect = self.pmap["presentation"]["block"]
result = self.mgr.presentation("block")
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
self.mgr.init()
system = "test"
rec = {
"notations": [
{"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
{"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""},
],
"system": system,
"type": "notation",
}
self.mgr.add(rec)
expect = rec["notations"]
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = Connection(dburi)
conn.drop_database(dbname)
coll = conn[dbname][collname]
self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
{"ui": "Block size", "das": "block.size"}]}}
coll.insert(self.pmap)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
self.mgr.create_db()
apiversion = 'DBS_2_0_8'
url = 'http://a.com'
dformat = 'JSON'
expire = 100
api = 'listRuns'
params = { 'apiversion':apiversion, 'path' : 'required', 'api':api}
rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url,
'params': params, 'expire':expire, "wild_card": "*",
'daskeys' : [dict(key='run', map='run.run_number', pattern='')],
'das2api' : [
dict(api_param='path', das_key='dataset', pattern=""),
]
}
self.mgr.add(rec)
res = self.mgr.check_dasmap('dbs', api, 'run.bfield')
self.assertEqual(False, res)
res = self.mgr.check_dasmap('dbs', api, 'run.run_number')
self.assertEqual(True, res)
smap = {api: {'url':url, 'expire':expire, 'keys': ['run'],
'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None,
'params': {'path': 'required', 'api': api,
'apiversion': 'DBS_2_0_8'}
}
}
rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat,
'url':url, 'expire': expire,
'params' : {'apiversion': apiversion, 'api': 'listBlocks',
'block_name':'*', 'storage_element_name':'*',
'user_type':'NORMAL'},
'daskeys': [
{'key':'block', 'map':'block.name', 'pattern':''},
],
'das2api': [
{'api_param':'storage_element_name',
'das_key':'site',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'storage_element_name',
'das_key':'site.se',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'block_name',
'das_key':'block',
'pattern':""},
{'api_param':'block_name',
'das_key':'block.name',
'pattern':""},
]
}
self.mgr.add(rec)
system = 'dbs'
api = 'listBlocks'
daskey = 'block'
primkey = 'block.name'
api_input = 'block_name'
res = self.mgr.list_systems()
self.assertEqual(['dbs'], res)
res = self.mgr.list_apis()
# self.assertEqual([api], res)
res.sort()
self.assertEqual(['listBlocks', 'listRuns'], res)
res = self.mgr.lookup_keys(system, daskey)
self.assertEqual([primkey], res)
value = ''
res = self.mgr.das2api(system, daskey, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, daskey, value, api)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey, primkey], res)
# adding notations
notations = {'system':system,
'notations':[
{'notation':'storage_element_name', 'map':'se', 'api':''},
{'notation':'number_of_events', 'map':'nevents', 'api':''},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual([daskey], res)
# build service map
smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
'keys': ['block'], 'format':dformat, "wild_card": "*",
'params': {'storage_element_name': '*', 'api':api,
'block_name': '*', 'user_type': 'NORMAL',
'apiversion': 'DBS_2_0_8'}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
self.mgr.create_db()
# rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}}
# self.mgr.add(rec)
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
self.mgr.create_db()
system = "test"
rec = {'notations': [
{"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""},
{"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""},
{"notation": "admin.contacts.cms_name", "map":"site.name", "api":""}
], "system": system}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi,
dbname=dbname,
collname=collname)
# add some maps to mapping db
conn = MongoClient(dburi)
conn.drop_database(dbname)
self.coll = conn[dbname][collname]
self.pmap = {
"presentation": {
"block": [{
"ui": "Block name",
"das": "block.name"
}, {
"ui": "Block size",
"das": "block.size"
}]
},
"type": "presentation"
}
self.coll.insert(self.pmap)
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
dformat = 'JSON'
system = 'dbs3'
expire = 100
rec = {
'system':
system,
'urn':
'acquisitioneras',
'format':
dformat,
'instances': ['prod/global'],
'url':
url,
'expire':
expire,
'lookup':
'era',
'params': {},
'das_map': [{
"das_key": "era",
"rec_key": "era.name",
"api_arg": "era"
}],
'type':
'service'
}
self.coll.insert(rec)
ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
rec = {'verification_token': ver_token, 'type': 'verification_token'}
self.coll.insert(rec)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
system = 'dbs3'
url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
dformat = 'JSON'
expire = 100
instances = ["prod/global", "prod/phys01"]
api = 'primarydatasets'
params = {"primary_ds_name": "*"}
rec = {
'system':
system,
'urn':
api,
'format':
dformat,
'url':
url,
'instances':
instances,
'params':
params,
'expire':
expire,
'lookup':
'primary_dataset',
'wild_card':
'*',
'das_map': [
dict(das_key='primary_dataset',
rec_key='primary_dataset.name',
api_arg='primary_dataset')
],
'type':
'service'
}
self.mgr.add(rec)
smap = {
api: {
'url': url,
'expire': expire,
'keys': ['primary_dataset'],
'format': dformat,
'wild_card': '*',
'cert': None,
'ckey': None,
'services': '',
'lookup': 'primary_dataset',
'params': params
}
}
rec = {
'system':
system,
'urn':
'datasetaccesstypes',
'format':
dformat,
'instances':
instances,
'url':
url,
'expire':
expire,
'lookup':
'status',
'params': {
'status': '*'
},
'das_map': [{
"das_key": "status",
"rec_key": "status.name",
"api_arg": "status"
}],
'type':
'service'
}
self.mgr.add(rec)
api = 'datasetaccesstypes'
daskey = 'status'
rec_key = 'status.name'
api_input = 'status'
res = self.mgr.list_systems()
self.assertEqual([system], res)
res = self.mgr.list_apis()
res.sort()
self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)
res = self.mgr.lookup_keys(system, api, daskey)
self.assertEqual([rec_key], res)
value = ''
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, api, rec_key, value)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey], res)
# adding notations
notations = {
'system':
system,
'type':
'notation',
'notations': [
{
'api_output': 'storage_element_name',
'rec_key': 'se',
'api': ''
},
{
'api_output': 'number_of_events',
'rec_key': 'nevents',
'api': ''
},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual(['status'], res)
# build service map
smap.update({
api: {
'url': url,
'expire': expire,
'cert': None,
'ckey': None,
'keys': ['status'],
'format': dformat,
'wild_card': '*',
'services': '',
'lookup': daskey,
'params': {
"status": "*"
}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
system = "test"
rec = {
'notations': [{
"api_output": "site.resource_element.cms_name",
"rec_key": "site.name",
"api": ""
}, {
"api_output": "site.resource_pledge.cms_name",
"rec_key": "site.name",
"api": ""
}, {
"api_output": "admin.contacts.cms_name",
"rec_key": "site.name",
"api": ""
}],
"system":
system,
"type":
"notation"
}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)