def __init__(self, name, config): self.name = name try: self.verbose = config['verbose'] title = 'DASAbstactService_%s' % self.name self.logger = PrintManager(title, self.verbose) self.dasmapping = config['dasmapping'] self.write2cache = config.get('write_cache', True) self.multitask = config['das'].get('multitask', True) self.error_expire = config['das'].get('error_expire', 300) self.dbs_global = None # to be configured at run time self.dburi = config['mongodb']['dburi'] engine = config.get('engine', None) self.gfs = db_gridfs(self.dburi) except Exception as exc: print_exc(exc) raise Exception('fail to parse DAS config') # read key/cert info try: self.ckey, self.cert = get_key_cert() except Exception as exc: print_exc(exc) self.ckey = None self.cert = None if self.multitask: nworkers = config['das'].get('api_workers', 3) thr_weights = config['das'].get('thread_weights', []) for system_weight in thr_weights: system, weight = system_weight.split(':') if system == self.name: nworkers *= int(weight) # if engine: # thr_name = 'DASAbstractService:%s:PluginTaskManager' % self.name # self.taskmgr = PluginTaskManager(\ # engine, nworkers=nworkers, name=thr_name) # self.taskmgr.subscribe() # else: # thr_name = 'DASAbstractService:%s:TaskManager' % self.name # self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) thr_name = 'DASAbstractService:%s:TaskManager' % self.name self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) else: self.taskmgr = None self.map = {} # to be defined by data-service implementation self._keys = None # to be defined at run-time in self.keys self._params = None # to be defined at run-time in self.parameters self._notations = {} # to be defined at run-time in self.notations self.logger.info('initialized') # define internal cache manager to put 'raw' results into cache if 'rawcache' in config and config['rawcache']: self.localcache = config['rawcache'] else: msg = 'Undefined rawcache, please check your configuration' raise Exception(msg)
def setUp(self): """ set up DAS core module """ debug = 0 config = deepcopy(das_readconfig()) logger = PrintManager('TestDASCache', verbose=debug) config['logger'] = logger config['verbose'] = debug
def __init__(self, config): self.verbose = config['verbose'] self.logger = PrintManager('DASParserDB', self.verbose) self.dburi = config['mongodb']['dburi'] self.dbname = config['parserdb']['dbname'] self.sizecap = config['parserdb'].get('sizecap', 5 * 1024 * 1024) self.colname = config['parserdb']['collname'] msg = "DASParserCache::__init__ %s@%s" % (self.dburi, self.dbname) self.logger.info(msg) self.create_db()
def test_debug(self): "Test logger debug method" old_stdout = sys.stdout logger = PrintManager(self.name, verbose=2) sys.stdout = StringIO.StringIO() logger.debug('test') result = sys.stdout.getvalue() expect = 'DEBUG %s:%s test\n' % (self.name, funcname()) self.assertEqual(expect, result) sys.stdout = old_stdout
def test_info(self): "Test logger info method" old_stdout = sys.stdout logger = PrintManager(self.name, verbose=1) sys.stdout = StringIO.StringIO() logger.info('test') result = sys.stdout.getvalue() expect = 'INFO %s:%s test\n' % (self.name, funcname()) self.assertEqual(expect, result) sys.stdout = old_stdout
def test_warning(self): "Test logger warning method" old_stdout = sys.stdout logger = PrintManager(self.name) # verbose is irrelevant sys.stdout = StringIO.StringIO() logger.warning('test') result = sys.stdout.getvalue() expect = 'WARNING %s:%s test\n' % (self.name, funcname()) self.assertEqual(expect, result) sys.stdout = old_stdout
def test_error(self): "Test logger error method" old_stdout = sys.stdout logger = PrintManager(self.name) # verbose is irrelevant sys.stdout = StringIO.StringIO() logger.error('test') result = sys.stdout.getvalue() expect = 'ERROR %s:%s test\n' % (self.name, funcname()) self.assertEqual(expect, result) sys.stdout = old_stdout
def __init__(self, config=None): if not config: config = das_readconfig() self.dasmapping = DASMapping(config) if not self.dasmapping.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.dasservices = config['services'] self.daskeysmap = self.dasmapping.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item)
def setUp(self): """ set up DAS core module """ debug = 0 config = deepcopy(das_readconfig()) logger = PrintManager('TestDASMongocache', verbose=debug) config['logger'] = logger config['verbose'] = debug dburi = config['mongodb']['dburi'] connection = MongoClient(dburi) connection.drop_database('das') dasmapping = DASMapping(config) config['dasmapping'] = dasmapping self.dasmongocache = DASMongocache(config)
def __init__(self, config): self.verbose = config['verbose'] self.logger = PrintManager('DASKeyLearning', self.verbose) self.services = config['services'] self.dburi = config['mongodb']['dburi'] self.dbname = config['keylearningdb']['dbname'] self.colname = config['keylearningdb']['collname'] self.mapping = config['dasmapping'] msg = "%s@%s" % (self.dburi, self.dbname) self.logger.info(msg) self.das_son_manipulator = DAS_SONManipulator() index_list = [('system', ASCENDING), ('urn', ASCENDING), \ ('members', ASCENDING), ('stems', ASCENDING)] create_indexes(self.col, index_list)
def __init__(self, config=None, debug=0, nores=False, logger=None, engine=None, multitask=True): if config: dasconfig = config else: dasconfig = das_readconfig() verbose = dasconfig['verbose'] self.stdout = debug if isinstance(debug, int) and debug: self.verbose = debug dasconfig['verbose'] = debug else: self.verbose = verbose das_timer('DASCore::init', self.verbose) self.operators = das_operators() self.collect_wait_time = dasconfig['das'].get('collect_wait_time', 120) # set noresults option self.noresults = False if nores: dasconfig['write_cache'] = True self.noresults = nores self.init_expire = dasconfig['das'].get('init_expire', 5 * 60) self.multitask = dasconfig['das'].get('multitask', True) if debug or self.verbose: self.multitask = False # in verbose mode do not use multitask dasconfig['das']['multitask'] = False if not multitask: # explicitly call DASCore ctor self.multitask = False dasconfig['das']['multitask'] = False dasconfig['engine'] = engine if self.multitask: nworkers = dasconfig['das'].get('core_workers', 5) # if engine: # thr_name = 'DASCore:PluginTaskManager' # self.taskmgr = PluginTaskManager(\ # engine, nworkers=nworkers, name=thr_name) # self.taskmgr.subscribe() # else: # thr_name = 'DASCore:TaskManager' # self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) thr_name = 'DASCore:TaskManager' self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name) else: self.taskmgr = None if logger: self.logger = logger else: self.logger = PrintManager('DASCore', self.verbose) # define Mapping/Analytics/Parser in this order since Parser depends # on first two dasmapping = DASMapping(dasconfig) dasconfig['dasmapping'] = dasmapping self.mapping = dasmapping self.keylearning = DASKeyLearning(dasconfig) dasconfig['keylearning'] = self.keylearning # init DAS cache self.rawcache = DASMongocache(dasconfig) dasconfig['rawcache'] = self.rawcache # plug-in architecture: loop over registered data-services in # dasconfig; load appropriate module/class; register data # service with DASCore. self.systems = dasmapping.list_systems() # pointer to the DAS top level directory dasroot = '/'.join(__file__.split('/')[:-3]) for name in self.systems: try: klass = 'DAS/services/%s/%s_service.py' \ % (name, name) srvfile = os.path.join(dasroot, klass) with open(srvfile) as srvclass: for line in srvclass: if line.find('(DASAbstractService)') != -1: klass = line.split('(DASAbstractService)')[0] klass = klass.split('class ')[-1] break mname = 'DAS.services.%s.%s_service' % (name, name) module = __import__(mname, fromlist=[klass]) obj = getattr(module, klass)(dasconfig) setattr(self, name, obj) except IOError as err: if debug > 1: # we have virtual services, so IOError can be correct print_exc(err) try: mname = 'DAS.services.generic_service' module = __import__(mname, fromlist=['GenericService']) obj = module.GenericService(name, dasconfig) setattr(self, name, obj) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) except Exception as exc: print_exc(exc) msg = "Unable to load %s data-service plugin" % name raise Exception(msg) # loop over systems and get system keys, add mapping keys to final list self.service_keys = {} self.service_parameters = {} for name in self.systems: skeys = list(getattr(self, name).keys()) self.service_keys[getattr(self, name).name] = skeys sparams = getattr(self, name).parameters() self.service_parameters[getattr(self, name).name] = sparams self.service_keys['special'] = das_special_keys() self.dasconfig = dasconfig das_timer('DASCore::init', self.verbose)
def __init__(self, config): self.config = config self.emptyset_expire = \ expire_timestamp(config['das'].get('emptyset_expire', 5)) self.dburi = config['mongodb']['dburi'] self.cache_size = config['mongodb']['bulkupdate_size'] self.dbname = config['dasdb']['dbname'] self.verbose = config['verbose'] self.logger = PrintManager('DASMongocache', self.verbose) self.mapping = config['dasmapping'] self.logging = config['dasdb'].get('logging', False) self.rec_ttl = config['dasdb'].get('record_ttl', 24 * 60 * 60) self.del_ttl = config['dasdb'].get('delta_ttl', 60) self.cleanup_del_ttl = config['dasdb'].get('cleanup_delta_ttl', 3600) self.retry = config['dasdb'].get('retry', 3) self.das_son_manipulator = DAS_SONManipulator() # Initialize MongoDB connection self.col_ = self.config['dasdb']['cachecollection'] self.mrcol_ = self.config['dasdb']['mrcollection'] self.merge_ = self.config['dasdb']['mergecollection'] self.gfs = db_gridfs(self.dburi) msg = "%s@%s" % (self.dburi, self.dbname) self.logger.info(msg) # ensure that we have the following indexes common_idx = [ ('file.name', DESCENDING), ('dataset.name', DESCENDING), ('block.name', DESCENDING), ('run.run_number', DESCENDING), ] index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING), ('das.system', ASCENDING), ('qhash', DESCENDING), ('das.record', ASCENDING)] create_indexes(self.col, index_list + common_idx) index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING), ('qhash', DESCENDING), ('das.record', ASCENDING), ('das.ts', ASCENDING)] create_indexes(self.merge, index_list) # NOTE: I found that creating index in merge collection leads to # MongoDB error when records contains multiple arrays on indexed # keys. For example, when we query file,run,lumi both file and run # are arrays in MongoDB. In this case the final sort in MongoDB # bark with the following message: # cannot sort with keys that are parallel arrays # it looks like that there is no fix for that yet # see # http://stackoverflow.com/questions/6516725/how-do-i-index-two-arrays-in-mongodb # therefore I temporary disabled create_indexes call on merge # collection which was used to have index to ease final sort, # especially in a case when a lot of records correspond to inital # query, e.g. file records. # On another hand, the most common use case where sort fails is # getting file records, and I can add one compound key to ease sort # but I can't add another compound key on array field, e.g. run common_idx = [[('qhash', DESCENDING), ('file.name', DESCENDING)]] create_indexes(self.merge, index_list + common_idx) # thread which clean-up DAS collections thname = 'mongocache_cleanup' cols = [ config['dasdb']['cachecollection'], config['dasdb']['mrcollection'], config['dasdb']['mergecollection'] ]
def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = { "presentation": { "block": [{ "ui": "Block name", "das": "block.name" }, { "ui": "Block size", "das": "block.size" }] }, "type": "presentation" } self.coll.insert(self.pmap) url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/' dformat = 'JSON' system = 'dbs3' expire = 100 rec = { 'system': system, 'urn': 'acquisitioneras', 'format': dformat, 'instances': ['prod/global'], 'url': url, 'expire': expire, 'lookup': 'era', 'params': {}, 'das_map': [{ "das_key": "era", "rec_key": "era.name", "api_arg": "era" }], 'type': 'service' } self.coll.insert(rec) ver_token = verification_token(self.coll.find(**PYMONGO_OPTS)) rec = {'verification_token': ver_token, 'type': 'verification_token'} self.coll.insert(rec) self.mgr = DASMapping(config)