class testCMSFakeDataServices(unittest.TestCase):
"""
A test class for the DAS core module
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
# read DAS config and make fake Mapping DB entry
collname = 'test_collection'
self.dasmerge = 'test_merge'
self.dascache = 'test_cache'
self.dasmr = 'test_mapreduce'
self.collname = collname
# config = deepcopy(das_readconfig())
config = das_readconfig()
dburi = config['mongodb']['dburi']
self.dburi = dburi
logger = PrintManager('TestCMSFakeDataServices', verbose=debug)
self.base = 'http://localhost:8080' # URL of DASTestDataService
self.expire = 100
config['logger'] = logger
config['loglevel'] = debug
config['verbose'] = debug
config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname)
config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100)
config['dasdb'] = {'dbname': 'das',
'cachecollection': self.dascache,
'mrcollection': self.dasmr,
'mergecollection': self.dasmerge}
config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'}
config['parserdb'] = {'collname': collname, 'dbname': 'parser',
'enable': True, 'sizecap': 10000}
config['services'] = ['dbs', 'phedex', 'sitedb', 'google_maps', 'ip']
# mongo parser
self.mongoparser = ql_manager(config)
config['mongoparser'] = self.mongoparser
# setup DAS mapper
self.mgr = DASMapping(config)
# create fresh DB
self.clear_collections()
self.mgr.delete_db_collection()
self.mgr.create_db()
# Add fake mapping records
self.add_service('ip', 'ip.yml')
self.add_service('google_maps', 'google_maps.yml')
self.add_service('dbs', 'dbs.yml')
self.add_service('phedex', 'phedex.yml')
self.add_service('sitedb', 'sitedb.yml')
# create DAS handler
self.das = DASCore(config)
# start TestDataService
self.server = Root(config)
self.server.start()
def add_service(self, system, ymlfile):
"""
Add Fake data service mapping records. We provide system name
which match corresponding name in DASTestDataService and
associated with this system YML map file.
"""
fname = os.path.join(DASPATH, 'services/maps/%s' % ymlfile)
url = self.base + '/%s' % system
for record in read_service_map(fname):
record['url'] = url
record['system'] = system
self.mgr.add(record)
for record in read_service_map(fname, 'notations'):
record['system'] = system
self.mgr.add(record)
def clear_collections(self):
"""clean-up test collections"""
conn = Connection(host=self.dburi)
for dbname in ['mapping', 'analytics', 'das', 'parser', 'keylearning']:
db = conn[dbname]
if dbname != 'das':
db.drop_collection(self.collname)
else:
db.drop_collection(self.dascache)
db.drop_collection(self.dasmerge)
db.drop_collection(self.dasmr)
def tearDown(self):
"""Invoke after each test"""
self.server.stop()
# self.mgr.delete_db_collection()
# self.clear_collections()
def testDBSService(self):
"""test DASCore with test DBS service"""
query = "primary_dataset=abc" # invoke query to fill DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "primary_dataset=abc" # invoke query to get results from DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('primary_dataset.name')
expect = 'abc'
self.assertEqual(expect, result)
def testPhedexAndSiteDBServices(self):
"""test DASCore with test PhEDEx and SiteDB services"""
query = "site=T3_US_Cornell" # invoke query to fill DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "site=T3_US_Cornell | grep site.name" # invoke query to get results from DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = 'T3_US_Cornell'
self.assertEqual(expect, DotDict(result[0]).get('site.name'))
expect = ['_id', 'das_id', 'site', 'cache_id', 'das', 'qhash']
expect.sort()
rkeys = result[0].keys()
rkeys.sort()
self.assertEqual(expect, rkeys)
def testAggregators(self):
"""test DASCore aggregators via zip service"""
query = "zip=1000"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "zip=1000 | count(zip.place.city)"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = {"function": "count", "result": {"value": 2},
"key": "zip.place.city", "_id":0}
self.assertEqual(expect, result[0])
def testIPService(self):
"""test DASCore with IP service"""
query = "ip=137.138.141.145"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "ip=137.138.141.145 | grep ip.address"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('ip.address')
expect = '137.138.141.145'
self.assertEqual(expect, result)
def testRecords(self):
"""test records DAS keyword with all services"""
query = "ip=137.138.141.145"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "site=T3_US_Cornell"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "records | grep ip.address"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('ip.address')
expect = '137.138.141.145'
self.assertEqual(expect, result)
query = "records | grep site.name"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = 'T3_US_Cornell'
self.assertEqual(expect, DotDict(result[0]).get('site.name'))
query = "records"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
res = []
for row in result:
if row.has_key('ip'):
res.append(DotDict(row).get('ip.address'))
if row.has_key('site'):
for item in row['site']:
if item.has_key('name') and item['name'] not in res:
res.append(item['name'])
res.sort()
expect = ['137.138.141.145', 'T3_US_Cornell']
self.assertEqual(expect, res)
def main():
"Main function"
optmgr = DASOptionParser()
(opts, _args) = optmgr.getOpt()
dburi = 'mongodb://%s:%s' % (opts.host, opts.port)
dasconfig = das_readconfig()
dbname, colname = opts.db.split('.')
mongodb = dict(dburi=dburi)
mappingdb = dict(dbname=dbname, collname=colname)
config = dict(verbose=opts.debug, mappingdb=mappingdb,
mongodb=mongodb, services=dasconfig['das'].get('services', []))
mgr = DASMapping(config)
if opts.listapis:
apis = mgr.list_apis(opts.system)
print apis
sys.exit(0)
if opts.listkeys:
keys = mgr.daskeys(opts.system)
print keys
sys.exit(0)
if opts.umap:
for rec in read_service_map(opts.umap, field='uri'):
if opts.debug:
print rec
spec = {'url':rec['url'], 'urn':rec['urn']}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.nmap:
for rec in read_service_map(opts.nmap, field='notations'):
if opts.debug:
print rec
system = rec['system']
spec = {'notations':{'$exists':True}, 'system':system}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.pmap:
for rec in read_service_map(opts.pmap, field='presentation'):
if opts.debug:
print rec
spec = {'presentation':{'$exists':True}}
mgr.remove(spec) # remove previous record
mgr.add(rec)
if opts.clean:
mgr.delete_db()
mgr.create_db()
# I need to clear DAS cache/merge since I don't know
# a-priory what kind of changes new maps will bring
conn = db_connection(dburi)
dbname = dasconfig['dasdb']['dbname']
cache = conn[dbname][dasconfig['dasdb']['cachecollection']]
cache.remove({})
merge = conn[dbname][dasconfig['dasdb']['mergecollection']]
merge.remove({})
if opts.remove:
mgr.remove(opts.remove)
class testDASMapping(unittest.TestCase):
"""
A test class for the DAS mappingdb class
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
self.db = 'test_mapping.db'
config = deepcopy(das_readconfig())
dburi = config['mongodb']['dburi']
logger = PrintManager('TestDASMapping', verbose=debug)
config['logger'] = logger
config['verbose'] = debug
dbname = 'test_mapping'
collname = 'db'
config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
# add some maps to mapping db
conn = Connection(dburi)
conn.drop_database(dbname)
coll = conn[dbname][collname]
self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
{"ui": "Block size", "das": "block.size"}]}}
coll.insert(self.pmap)
self.mgr = DASMapping(config)
def tearDown(self):
"""Invoke after each test"""
self.mgr.delete_db()
def test_api(self):
"""test methods for api table"""
self.mgr.delete_db()
self.mgr.create_db()
apiversion = 'DBS_2_0_8'
url = 'http://a.com'
dformat = 'JSON'
expire = 100
api = 'listRuns'
params = { 'apiversion':apiversion, 'path' : 'required', 'api':api}
rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url,
'params': params, 'expire':expire, "wild_card": "*",
'daskeys' : [dict(key='run', map='run.run_number', pattern='')],
'das2api' : [
dict(api_param='path', das_key='dataset', pattern=""),
]
}
self.mgr.add(rec)
res = self.mgr.check_dasmap('dbs', api, 'run.bfield')
self.assertEqual(False, res)
res = self.mgr.check_dasmap('dbs', api, 'run.run_number')
self.assertEqual(True, res)
smap = {api: {'url':url, 'expire':expire, 'keys': ['run'],
'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None,
'params': {'path': 'required', 'api': api,
'apiversion': 'DBS_2_0_8'}
}
}
rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat,
'url':url, 'expire': expire,
'params' : {'apiversion': apiversion, 'api': 'listBlocks',
'block_name':'*', 'storage_element_name':'*',
'user_type':'NORMAL'},
'daskeys': [
{'key':'block', 'map':'block.name', 'pattern':''},
],
'das2api': [
{'api_param':'storage_element_name',
'das_key':'site',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'storage_element_name',
'das_key':'site.se',
'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
{'api_param':'block_name',
'das_key':'block',
'pattern':""},
{'api_param':'block_name',
'das_key':'block.name',
'pattern':""},
]
}
self.mgr.add(rec)
system = 'dbs'
api = 'listBlocks'
daskey = 'block'
primkey = 'block.name'
api_input = 'block_name'
res = self.mgr.list_systems()
self.assertEqual(['dbs'], res)
res = self.mgr.list_apis()
# self.assertEqual([api], res)
res.sort()
self.assertEqual(['listBlocks', 'listRuns'], res)
res = self.mgr.lookup_keys(system, daskey)
self.assertEqual([primkey], res)
value = ''
res = self.mgr.das2api(system, daskey, value)
self.assertEqual([api_input], res)
# adding another params which default is None
res = self.mgr.das2api(system, daskey, value, api)
self.assertEqual([api_input], res)
res = self.mgr.api2das(system, api_input)
self.assertEqual([daskey, primkey], res)
# adding notations
notations = {'system':system,
'notations':[
{'notation':'storage_element_name', 'map':'se', 'api':''},
{'notation':'number_of_events', 'map':'nevents', 'api':''},
]
}
self.mgr.add(notations)
res = self.mgr.notation2das(system, 'number_of_events')
self.assertEqual('nevents', res)
# API keys
res = self.mgr.api2daskey(system, api)
self.assertEqual([daskey], res)
# build service map
smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
'keys': ['block'], 'format':dformat, "wild_card": "*",
'params': {'storage_element_name': '*', 'api':api,
'block_name': '*', 'user_type': 'NORMAL',
'apiversion': 'DBS_2_0_8'}
}
})
res = self.mgr.servicemap(system)
self.assertEqual(smap, res)
def test_presentation(self):
"""test presentation method"""
self.mgr.create_db()
# rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}}
# self.mgr.add(rec)
expect = self.pmap['presentation']['block']
result = self.mgr.presentation('block')
self.assertEqual(expect, result)
def test_notations(self):
"""test notations method"""
self.mgr.create_db()
system = "test"
rec = {'notations': [
{"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""},
{"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""},
{"notation": "admin.contacts.cms_name", "map":"site.name", "api":""}
], "system": system}
self.mgr.add(rec)
expect = rec['notations']
result = self.mgr.notations(system)[system]
self.assertEqual(expect, result)
