def setUp(self):
"""
set up shared array of type int with given size
"""
self.size = 10
self.data = Array('i', range(self.size))
# start TestDataService
self.server = Root()
def setUp(self):
"""
set up DAS core module
"""
debug = 0
# read DAS config and make fake Mapping DB entry
collname = 'test_collection'
self.dasmerge = 'test_merge'
self.dascache = 'test_cache'
self.dasmr = 'test_mapreduce'
self.collname = collname
# config = deepcopy(das_readconfig())
config = das_readconfig()
dburi = config['mongodb']['dburi']
self.dburi = dburi
logger = PrintManager('TestCMSFakeDataServices', verbose=debug)
self.base = 'http://localhost:8080' # URL of DASTestDataService
self.expire = 100
config['logger'] = logger
config['loglevel'] = debug
config['verbose'] = debug
config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname)
config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100)
config['dasdb'] = {'dbname': 'das',
'cachecollection': self.dascache,
'mrcollection': self.dasmr,
'mergecollection': self.dasmerge}
config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'}
config['parserdb'] = {'collname': collname, 'dbname': 'parser',
'enable': True, 'sizecap': 10000}
config['services'] = ['dbs', 'phedex', 'sitedb', 'google_maps', 'ip']
# mongo parser
self.mongoparser = ql_manager(config)
config['mongoparser'] = self.mongoparser
# setup DAS mapper
self.mgr = DASMapping(config)
# create fresh DB
self.clear_collections()
self.mgr.delete_db_collection()
self.mgr.create_db()
# Add fake mapping records
self.add_service('ip', 'ip.yml')
self.add_service('google_maps', 'google_maps.yml')
self.add_service('dbs', 'dbs.yml')
self.add_service('phedex', 'phedex.yml')
self.add_service('sitedb', 'sitedb.yml')
# create DAS handler
self.das = DASCore(config)
# start TestDataService
self.server = Root(config)
self.server.start()
def setUp(self):
"""
set up shared array of type int with given size
"""
self.size = 10
self.data = Array('i', range(self.size))
# start TestDataService
self.server = Root()
def setUp(self):
"""
set up DAS core module
"""
debug = 0
# read DAS config and make fake Mapping DB entry
collname = 'test_collection'
self.dasmerge = 'test_merge'
self.dascache = 'test_cache'
self.dasmr = 'test_mapreduce'
self.collname = collname
config = das_readconfig()
dburi = config['mongodb']['dburi']
self.dburi = dburi
logger = PrintManager('TestCMSFakeDataServices', verbose=debug)
self.base = 'http://127.0.0.1:8080' # URL of DASTestDataService
self.expire = 100
config['logger'] = logger
config['loglevel'] = debug
config['verbose'] = debug
config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname)
config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100)
config['dasdb'] = {'dbname': 'das',
'cachecollection': self.dascache,
'mrcollection': self.dasmr,
'mergecollection': self.dasmerge}
config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'}
config['parserdb'] = {'collname': collname, 'dbname': 'parser',
'enable': True, 'sizecap': 10000}
config['services'] = ['dbs3', 'phedex', 'google_maps', 'ip']
# Do not perform DAS map test, since we overwrite system and urls.
# This is done to use standard DAS maps, but use local URLs, which
# cause DAS hash map to be be wrong during a test
config['map_test'] = False
# Add fake mapping records
self.clear_collections()
self.add_service('ip', 'ip.yml')
self.add_service('google_maps', 'google_maps.yml')
self.add_service('dbs3', 'dbs3.yml')
self.add_service('phedex', 'phedex.yml')
# setup DAS mapper
self.mgr = DASMapping(config)
# mongo parser
self.mongoparser = ql_manager(config)
config['mongoparser'] = self.mongoparser
# create DAS handler
self.das = DASCore(config)
# start TestDataService
self.server = Root(config)
self.server.start()
class testUtils(unittest.TestCase):
"""
A test class for the DAS task manager class
"""
def setUp(self):
"""
set up shared array of type int with given size
"""
self.size = 10
self.data = Array('i', range(self.size))
# start TestDataService
self.server = Root()
def test_task_manager(self):
"""Test task manager"""
expect = [idx for idx in range(self.size)]
mypool = TaskManager()
tasks = []
for idx in expect:
tasks.append(mypool.spawn(worker, idx, self.data))
mypool.joinall(tasks)
result = [idx for idx in self.data]
self.assertEqual(result, expect)
def test_plugin_task_manager(self):
"""Test plugin task manager"""
mgr = PluginTaskManager(bus=self.server.engine, debug=1)
mgr.subscribe()
self.server.start()
jobs = []
jobs.append(mgr.spawn(daemon))
mgr.clear(jobs)
print "\njoin the task at %s\n" % time.time()
time.sleep(2)
print "\nstop server at %s\n" % time.time()
self.server.stop()
class testUtils(unittest.TestCase):
"""
A test class for the DAS task manager class
"""
def setUp(self):
"""
set up shared array of type int with given size
"""
self.size = 10
self.data = Array('i', range(self.size))
# start TestDataService
self.server = Root()
def test_task_manager(self):
"""Test task manager"""
expect = [idx for idx in range(self.size)]
mypool = TaskManager()
tasks = []
for idx in expect:
tasks.append(mypool.spawn(worker, idx, self.data))
mypool.joinall(tasks)
result = [idx for idx in self.data]
self.assertEqual(result, expect)
def test_plugin_task_manager(self):
"""Test plugin task manager"""
mgr = PluginTaskManager(bus=self.server.engine, debug=1)
mgr.subscribe()
self.server.start()
jobs = []
jobs.append(mgr.spawn(daemon))
mgr.clear(jobs)
time.sleep(2) # let jobs finish
self.server.stop()
def test_uidset(self):
"""Test UidSet class"""
tasks = UidSet()
tasks.add(1)
self.assertEqual(1 in tasks, True)
self.assertEqual(2 in tasks, False)
tasks.add(1)
tasks.add(2)
self.assertEqual(1 in tasks, True)
self.assertEqual(2 in tasks, True)
self.assertEqual(tasks.get(1), 2) # we should have 2 values of 1
self.assertEqual(tasks.get(2), 1) # we should have 1 value of 2
tasks.discard(2)
self.assertEqual(2 in tasks, False)
tasks.discard(1)
self.assertEqual(tasks.get(1), 1) # now we should have 1 value of 1
tasks.discard(1)
self.assertEqual(1 in tasks, False)
def test_assign_priority(self):
"""Test priority assignment"""
tasks = TaskManager(qtype='PriorityQueue')
uid1 = '1.1.1.1'
tasks._uids.add(uid1)
uid2 = '2.2.2.2'
tasks._uids.add(uid1)
result = tasks.assign_priority(uid1) # no tasks in a queue
self.assertEqual(result, 0)
tasks._tasks = TestQueue(empty=False)
res1 = [tasks._uids.add(uid1) for r in xrange(20)]
self.assertEqual(tasks.assign_priority(uid1), 2)
res2 = [tasks._uids.add(uid2) for r in xrange(50)]
self.assertEqual(tasks.assign_priority(uid2), 5)
def test_priority_task_manager(self):
"""Test priority task manager"""
data = [idx for idx in xrange(0, 100)]
shared_data = Array('i', len(data))
mypool = TaskManager(qtype='PriorityQueue')
tasks = []
for idx in data:
if idx%2:
tasks.append(mypool.spawn(worker, idx, shared_data, uid=1))
else:
tasks.append(mypool.spawn(worker, idx, shared_data, uid=2))
mypool.joinall(tasks)
result = [idx for idx in shared_data]
self.assertEqual(result, data)
class testCMSFakeDataServices(unittest.TestCase):
"""
A test class for the DAS core module
"""
def setUp(self):
"""
set up DAS core module
"""
debug = 0
# read DAS config and make fake Mapping DB entry
collname = 'test_collection'
self.dasmerge = 'test_merge'
self.dascache = 'test_cache'
self.dasmr = 'test_mapreduce'
self.collname = collname
# config = deepcopy(das_readconfig())
config = das_readconfig()
dburi = config['mongodb']['dburi']
self.dburi = dburi
logger = PrintManager('TestCMSFakeDataServices', verbose=debug)
self.base = 'http://localhost:8080' # URL of DASTestDataService
self.expire = 100
config['logger'] = logger
config['loglevel'] = debug
config['verbose'] = debug
config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname)
config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100)
config['dasdb'] = {'dbname': 'das',
'cachecollection': self.dascache,
'mrcollection': self.dasmr,
'mergecollection': self.dasmerge}
config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'}
config['parserdb'] = {'collname': collname, 'dbname': 'parser',
'enable': True, 'sizecap': 10000}
config['services'] = ['dbs', 'phedex', 'sitedb', 'google_maps', 'ip']
# mongo parser
self.mongoparser = ql_manager(config)
config['mongoparser'] = self.mongoparser
# setup DAS mapper
self.mgr = DASMapping(config)
# create fresh DB
self.clear_collections()
self.mgr.delete_db_collection()
self.mgr.create_db()
# Add fake mapping records
self.add_service('ip', 'ip.yml')
self.add_service('google_maps', 'google_maps.yml')
self.add_service('dbs', 'dbs.yml')
self.add_service('phedex', 'phedex.yml')
self.add_service('sitedb', 'sitedb.yml')
# create DAS handler
self.das = DASCore(config)
# start TestDataService
self.server = Root(config)
self.server.start()
def add_service(self, system, ymlfile):
"""
Add Fake data service mapping records. We provide system name
which match corresponding name in DASTestDataService and
associated with this system YML map file.
"""
fname = os.path.join(DASPATH, 'services/maps/%s' % ymlfile)
url = self.base + '/%s' % system
for record in read_service_map(fname):
record['url'] = url
record['system'] = system
self.mgr.add(record)
for record in read_service_map(fname, 'notations'):
record['system'] = system
self.mgr.add(record)
def clear_collections(self):
"""clean-up test collections"""
conn = Connection(host=self.dburi)
for dbname in ['mapping', 'analytics', 'das', 'parser', 'keylearning']:
db = conn[dbname]
if dbname != 'das':
db.drop_collection(self.collname)
else:
db.drop_collection(self.dascache)
db.drop_collection(self.dasmerge)
db.drop_collection(self.dasmr)
def tearDown(self):
"""Invoke after each test"""
self.server.stop()
# self.mgr.delete_db_collection()
# self.clear_collections()
def testDBSService(self):
"""test DASCore with test DBS service"""
query = "primary_dataset=abc" # invoke query to fill DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "primary_dataset=abc" # invoke query to get results from DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('primary_dataset.name')
expect = 'abc'
self.assertEqual(expect, result)
def testPhedexAndSiteDBServices(self):
"""test DASCore with test PhEDEx and SiteDB services"""
query = "site=T3_US_Cornell" # invoke query to fill DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "site=T3_US_Cornell | grep site.name" # invoke query to get results from DAS cache
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = 'T3_US_Cornell'
self.assertEqual(expect, DotDict(result[0]).get('site.name'))
expect = ['_id', 'das_id', 'site', 'cache_id', 'das', 'qhash']
expect.sort()
rkeys = result[0].keys()
rkeys.sort()
self.assertEqual(expect, rkeys)
def testAggregators(self):
"""test DASCore aggregators via zip service"""
query = "zip=1000"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "zip=1000 | count(zip.place.city)"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = {"function": "count", "result": {"value": 2},
"key": "zip.place.city", "_id":0}
self.assertEqual(expect, result[0])
def testIPService(self):
"""test DASCore with IP service"""
query = "ip=137.138.141.145"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "ip=137.138.141.145 | grep ip.address"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('ip.address')
expect = '137.138.141.145'
self.assertEqual(expect, result)
def testRecords(self):
"""test records DAS keyword with all services"""
query = "ip=137.138.141.145"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "site=T3_US_Cornell"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.call(dquery)
expect = "ok"
self.assertEqual(expect, result)
query = "records | grep ip.address"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
result = DotDict(result[0]).get('ip.address')
expect = '137.138.141.145'
self.assertEqual(expect, result)
query = "records | grep site.name"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
result = [r for r in result]
expect = 'T3_US_Cornell'
self.assertEqual(expect, DotDict(result[0]).get('site.name'))
query = "records"
dquery = DASQuery(query, mongoparser=self.mongoparser)
result = self.das.get_from_cache(dquery, collection=self.dasmerge)
res = []
for row in result:
if row.has_key('ip'):
res.append(DotDict(row).get('ip.address'))
if row.has_key('site'):
for item in row['site']:
if item.has_key('name') and item['name'] not in res:
res.append(item['name'])
res.sort()
expect = ['137.138.141.145', 'T3_US_Cornell']
self.assertEqual(expect, res)
class testUtils(unittest.TestCase):
"""
A test class for the DAS task manager class
"""
def setUp(self):
"""
set up shared array of type int with given size
"""
self.size = 10
self.data = Array('i', range(self.size))
# start TestDataService
self.server = Root()
def test_task_manager(self):
"""Test task manager"""
expect = [idx for idx in range(self.size)]
mypool = TaskManager()
tasks = []
for idx in expect:
tasks.append(mypool.spawn(worker, idx, self.data))
mypool.joinall(tasks)
result = [idx for idx in self.data]
self.assertEqual(result, expect)
def test_plugin_task_manager(self):
"""Test plugin task manager"""
mgr = PluginTaskManager(bus=self.server.engine, debug=1)
mgr.subscribe()
self.server.start()
jobs = []
jobs.append(mgr.spawn(daemon))
mgr.clear(jobs)
time.sleep(2) # let jobs finish
self.server.stop()
def test_uidset(self):
"""Test UidSet class"""
tasks = UidSet()
tasks.add(1)
self.assertEqual(1 in tasks, True)
self.assertEqual(2 in tasks, False)
tasks.add(1)
tasks.add(2)
self.assertEqual(1 in tasks, True)
self.assertEqual(2 in tasks, True)
self.assertEqual(tasks.get(1), 2) # we should have 2 values of 1
self.assertEqual(tasks.get(2), 1) # we should have 1 value of 2
tasks.discard(2)
self.assertEqual(2 in tasks, False)
tasks.discard(1)
self.assertEqual(tasks.get(1), 1) # now we should have 1 value of 1
tasks.discard(1)
self.assertEqual(1 in tasks, False)
def test_assign_priority(self):
"""Test priority assignment"""
tasks = TaskManager(qtype='PriorityQueue', qfreq=10)
uid1 = '1.1.1.1'
tasks._uids.add(uid1)
uid2 = '2.2.2.2'
tasks._uids.add(uid1)
result = tasks.assign_priority(uid1) # no tasks in a queue
self.assertEqual(int(result), 0)
tasks._tasks = TestQueue(empty=False)
res1 = [tasks._uids.add(uid1) for r in range(20)]
self.assertEqual(int(tasks.assign_priority(uid1)), 2)
res2 = [tasks._uids.add(uid2) for r in range(50)]
self.assertEqual(int(tasks.assign_priority(uid2)), 5)
def test_priority_task_manager(self):
"""Test priority task manager"""
data = [idx for idx in range(0, 30)]
shared_data = Array('i', len(data))
mypool = TaskManager(qtype='PriorityQueue', qfreq=10)
tasks = []
for idx in data:
if idx % 2:
tasks.append(mypool.spawn(worker, idx, shared_data, uid=1))
else:
tasks.append(mypool.spawn(worker, idx, shared_data, uid=2))
mypool.joinall(tasks)
result = [idx for idx in shared_data]
self.assertEqual(result, data)
