def __init__(self, name, config):
self.name = name
try:
self.verbose = config['verbose']
title = 'DASAbstactService_%s' % self.name
self.logger = PrintManager(title, self.verbose)
self.dasmapping = config['dasmapping']
self.analytics = config['dasanalytics']
self.write2cache = config.get('write_cache', True)
self.multitask = config['das'].get('multitask', True)
self.error_expire = config['das'].get('error_expire', 300)
if config.has_key('dbs'):
self.dbs_global = config['dbs'].get('dbs_global_instance', None)
else:
self.dbs_global = None
dburi = config['mongodb']['dburi']
engine = config.get('engine', None)
self.gfs = db_gridfs(dburi)
except Exception as exc:
print_exc(exc)
raise Exception('fail to parse DAS config')
# read key/cert info
try:
self.ckey, self.cert = get_key_cert()
except Exception as exc:
print_exc(exc)
self.ckey = None
self.cert = None
if self.multitask:
nworkers = config['das'].get('api_workers', 3)
thr_weights = config['das'].get('thread_weights', [])
for system_weight in thr_weights:
system, weight = system_weight.split(':')
if system == self.name:
nworkers *= int(weight)
if engine:
thr_name = 'DASAbstractService:%s:PluginTaskManager' % self.name
self.taskmgr = PluginTaskManager(\
engine, nworkers=nworkers, name=thr_name)
self.taskmgr.subscribe()
else:
thr_name = 'DASAbstractService:%s:TaskManager' % self.name
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
else:
self.taskmgr = None
self.map = {} # to be defined by data-service implementation
self._keys = None # to be defined at run-time in self.keys
self._params = None # to be defined at run-time in self.parameters
self._notations = {} # to be defined at run-time in self.notations
self.logger.info('initialized')
# define internal cache manager to put 'raw' results into cache
if config.has_key('rawcache') and config['rawcache']:
self.localcache = config['rawcache']
else:
msg = 'Undefined rawcache, please check your configuration'
raise Exception(msg)
def init(self):
"""Init DAS web server, connect to DAS Core"""
try:
self.logcol = DASLogdb(self.dasconfig)
self.reqmgr = RequestManager(self.dburi, lifetime=self.lifetime)
self.dasmgr = DASCore(engine=self.engine)
self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr)
self.daskeys = self.dasmgr.das_keys()
self.gfs = db_gridfs(self.dburi)
self.daskeys.sort()
self.dasmapping = self.dasmgr.mapping
self.dasmapping.init_presentationcache()
self.colors = {}
for system in self.dasmgr.systems:
self.colors[system] = gen_color(system)
self.sitedbmgr = SiteDBService(self.dasconfig)
except Exception as exc:
print_exc(exc)
self.dasmgr = None
self.daskeys = []
self.colors = {}
return
# Start Onhold_request daemon
if self.dasconfig['web_server'].get('onhold_daemon', False):
self.process_requests_onhold()
def __init__(self, name, config):
self.name = name
try:
self.verbose = config['verbose']
title = 'DASAbstactService_%s' % self.name
self.logger = PrintManager(title, self.verbose)
self.dasmapping = config['dasmapping']
self.write2cache = config.get('write_cache', True)
self.multitask = config['das'].get('multitask', True)
self.error_expire = config['das'].get('error_expire', 300)
self.dbs_global = None # to be configured at run time
self.dburi = config['mongodb']['dburi']
engine = config.get('engine', None)
self.gfs = db_gridfs(self.dburi)
except Exception as exc:
print_exc(exc)
raise Exception('fail to parse DAS config')
# read key/cert info
try:
self.ckey, self.cert = get_key_cert()
except Exception as exc:
print_exc(exc)
self.ckey = None
self.cert = None
if self.multitask:
nworkers = config['das'].get('api_workers', 3)
thr_weights = config['das'].get('thread_weights', [])
for system_weight in thr_weights:
system, weight = system_weight.split(':')
if system == self.name:
nworkers *= int(weight)
# if engine:
# thr_name = 'DASAbstractService:%s:PluginTaskManager' % self.name
# self.taskmgr = PluginTaskManager(\
# engine, nworkers=nworkers, name=thr_name)
# self.taskmgr.subscribe()
# else:
# thr_name = 'DASAbstractService:%s:TaskManager' % self.name
# self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
thr_name = 'DASAbstractService:%s:TaskManager' % self.name
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
else:
self.taskmgr = None
self.map = {} # to be defined by data-service implementation
self._keys = None # to be defined at run-time in self.keys
self._params = None # to be defined at run-time in self.parameters
self._notations = {} # to be defined at run-time in self.notations
self.logger.info('initialized')
# define internal cache manager to put 'raw' results into cache
if 'rawcache' in config and config['rawcache']:
self.localcache = config['rawcache']
else:
msg = 'Undefined rawcache, please check your configuration'
raise Exception(msg)
def __init__(self, name, config):
self.name = name
try:
self.verbose = config["verbose"]
title = "DASAbstactService_%s" % self.name
self.logger = PrintManager(title, self.verbose)
self.dasmapping = config["dasmapping"]
self.write2cache = config.get("write_cache", True)
self.multitask = config["das"].get("multitask", True)
self.error_expire = config["das"].get("error_expire", 300)
self.dbs_global = None # to be configured at run time
self.dburi = config["mongodb"]["dburi"]
engine = config.get("engine", None)
self.gfs = db_gridfs(self.dburi)
except Exception as exc:
print_exc(exc)
raise Exception("fail to parse DAS config")
# read key/cert info
try:
self.ckey, self.cert = get_key_cert()
except Exception as exc:
print_exc(exc)
self.ckey = None
self.cert = None
if self.multitask:
nworkers = config["das"].get("api_workers", 3)
thr_weights = config["das"].get("thread_weights", [])
for system_weight in thr_weights:
system, weight = system_weight.split(":")
if system == self.name:
nworkers *= int(weight)
if engine:
thr_name = "DASAbstractService:%s:PluginTaskManager" % self.name
self.taskmgr = PluginTaskManager(engine, nworkers=nworkers, name=thr_name)
self.taskmgr.subscribe()
else:
thr_name = "DASAbstractService:%s:TaskManager" % self.name
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name)
else:
self.taskmgr = None
self.map = {} # to be defined by data-service implementation
self._keys = None # to be defined at run-time in self.keys
self._params = None # to be defined at run-time in self.parameters
self._notations = {} # to be defined at run-time in self.notations
self.logger.info("initialized")
# define internal cache manager to put 'raw' results into cache
if "rawcache" in config and config["rawcache"]:
self.localcache = config["rawcache"]
else:
msg = "Undefined rawcache, please check your configuration"
raise Exception(msg)
def test_db_gridfs(self):
"""Test db_gridfs"""
fsinst = db_gridfs(self.dburi)
doc = 'hello world!'
fid = fsinst.put(doc)
content = fsinst.get(fid).read()
self.assertEqual(doc, content)
fsinst.delete(fid)
rec = {'test': 1}
self.assertRaises(Exception, fsinst.put, rec)
def test_db_gridfs(self):
"""Test db_gridfs"""
fsinst = db_gridfs(self.dburi)
doc = 'hello world!'
fid = fsinst.put(doc)
content = fsinst.get(fid).read()
self.assertEqual(doc, content)
fsinst.delete(fid)
rec = {'test': 1}
self.assertRaises(Exception, fsinst.put, rec)
def init(self):
"""Init DAS web server, connect to DAS Core"""
try:
self.reqmgr = RequestManager(lifetime=self.lifetime)
self.dasmgr = DASCore(engine=self.engine)
self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr)
self.daskeys = self.dasmgr.das_keys()
self.gfs = db_gridfs(self.dburi)
self.daskeys.sort()
self.dasmapping = self.dasmgr.mapping
self.dbs_url = self.dasmapping.dbs_url()
self.dbs_global = self.dasmapping.dbs_global_instance()
self.dbs_instances = self.dasmapping.dbs_instances()
self.dasmapping.init_presentationcache()
self.colors = {"das": gen_color("das")}
for system in self.dasmgr.systems:
self.colors[system] = gen_color(system)
# get SiteDB from global scope
self.sitedbmgr = SERVICES.get("sitedb2", None)
# Start DBS daemon
if self.dataset_daemon:
self.dbs_daemon(self.dasconfig["web_server"])
if not self.daskeyslist:
keylist = [r for r in self.dasmapping.das_presentation_map()]
keylist.sort(key=lambda r: r["das"])
self.daskeyslist = keylist
except ConnectionFailure as _err:
tstamp = dastimestamp("")
mythr = threading.current_thread()
print "### MongoDB connection failure thread=%s, id=%s, time=%s" % (mythr.name, mythr.ident, tstamp)
except Exception as exc:
print_exc(exc)
self.dasmgr = None
self.reqmgr = None
self.dbs_url = None
self.dbs_global = None
self.dbs_instances = []
self.daskeys = []
self.colors = {}
self.q_rewriter = None
return
# KWS and Query Rewriting failures are not fatal
try:
# init query rewriter, if needed
if self.dasconfig["query_rewrite"]["pk_rewrite_on"]:
self.q_rewriter = CMSQueryRewrite(self.repmgr, self.templatepage)
except Exception as exc:
print_exc(exc)
self.q_rewriter = None
def init(self):
"""Init DAS web server, connect to DAS Core"""
try:
self.reqmgr = RequestManager(lifetime=self.lifetime)
self.dasmgr = DASCore(engine=self.engine)
self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr)
self.daskeys = self.dasmgr.das_keys()
self.gfs = db_gridfs(self.dburi)
self.daskeys.sort()
self.dasmapping = self.dasmgr.mapping
self.dbs_url = self.dasmapping.dbs_url()
self.dbs_global = self.dasmapping.dbs_global_instance()
self.dbs_instances = self.dasmapping.dbs_instances()
self.dasmapping.init_presentationcache()
self.colors = {'das':gen_color('das')}
for system in self.dasmgr.systems:
self.colors[system] = gen_color(system)
if not self.daskeyslist:
keylist = [r for r in self.dasmapping.das_presentation_map()]
keylist.sort(key=lambda r: r['das'])
self.daskeyslist = keylist
except ConnectionFailure as _err:
tstamp = dastimestamp('')
mythr = threading.current_thread()
print("### MongoDB connection failure thread=%s, id=%s, time=%s" \
% (mythr.name, mythr.ident, tstamp))
except Exception as exc:
print_exc(exc)
self.dasmgr = None
self.reqmgr = None
self.dbs_url = None
self.dbs_global = None
self.dbs_instances = []
self.daskeys = []
self.colors = {}
self.q_rewriter = None
return
# KWS and Query Rewriting failures are not fatal
try:
# init query rewriter, if needed
if self.dasconfig['query_rewrite']['pk_rewrite_on']:
self.q_rewriter = CMSQueryRewrite(self.repmgr,
self.templatepage)
except Exception as exc:
print_exc(exc)
self.q_rewriter = None
def __init__(self, config):
self.emptyset_expire = expire_timestamp(\
config['das'].get('emptyset_expire', 5))
self.dburi = config['mongodb']['dburi']
self.cache_size = config['mongodb']['bulkupdate_size']
self.dbname = config['dasdb']['dbname']
self.verbose = config['verbose']
self.logger = PrintManager('DASMongocache', self.verbose)
self.mapping = config['dasmapping']
self.conn = db_connection(self.dburi)
self.mdb = self.conn[self.dbname]
self.col = self.mdb[config['dasdb']['cachecollection']]
self.mrcol = self.mdb[config['dasdb']['mrcollection']]
self.merge = self.mdb[config['dasdb']['mergecollection']]
self.gfs = db_gridfs(self.dburi)
self.logdb = DASLogdb(config)
self.das_internal_keys = ['das_id', 'das', 'cache_id', 'qhash']
msg = "%s@%s" % (self.dburi, self.dbname)
self.logger.info(msg)
self.add_manipulator()
# ensure that we have the following indexes
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('das.system', ASCENDING),
('qhash', DESCENDING),
('das.empty_record', ASCENDING)]
create_indexes(self.col, index_list)
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('qhash', DESCENDING),
('das.empty_record', ASCENDING), ('das.ts', ASCENDING)]
create_indexes(self.merge, index_list)
def __init__(self, config):
self.config = config
self.emptyset_expire = \
expire_timestamp(config['das'].get('emptyset_expire', 5))
self.dburi = config['mongodb']['dburi']
self.cache_size = config['mongodb']['bulkupdate_size']
self.dbname = config['dasdb']['dbname']
self.verbose = config['verbose']
self.logger = PrintManager('DASMongocache', self.verbose)
self.mapping = config['dasmapping']
self.logging = config['dasdb'].get('logging', False)
self.rec_ttl = config['dasdb'].get('record_ttl', 24*60*60)
self.del_ttl = config['dasdb'].get('delta_ttl', 60)
self.cleanup_del_ttl = config['dasdb'].get('cleanup_delta_ttl', 3600)
self.retry = config['dasdb'].get('retry', 3)
self.das_son_manipulator = DAS_SONManipulator()
# Initialize MongoDB connection
self.col_ = self.config['dasdb']['cachecollection']
self.mrcol_ = self.config['dasdb']['mrcollection']
self.merge_ = self.config['dasdb']['mergecollection']
self.gfs = db_gridfs(self.dburi)
msg = "%s@%s" % (self.dburi, self.dbname)
self.logger.info(msg)
# ensure that we have the following indexes
common_idx = [
('file.name', DESCENDING),
('dataset.name', DESCENDING),
('block.name', DESCENDING),
('run.run_number', DESCENDING),
]
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('das.system', ASCENDING),
('qhash', DESCENDING),
('das.record', ASCENDING)]
create_indexes(self.col, index_list + common_idx)
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('qhash', DESCENDING),
('das.record', ASCENDING),
('das.ts', ASCENDING)]
create_indexes(self.merge, index_list)
# NOTE: I found that creating index in merge collection leads to
# MongoDB error when records contains multiple arrays on indexed
# keys. For example, when we query file,run,lumi both file and run
# are arrays in MongoDB. In this case the final sort in MongoDB
# bark with the following message:
# cannot sort with keys that are parallel arrays
# it looks like that there is no fix for that yet
# see
# http://stackoverflow.com/questions/6516725/how-do-i-index-two-arrays-in-mongodb
# therefore I temporary disabled create_indexes call on merge
# collection which was used to have index to ease final sort,
# especially in a case when a lot of records correspond to inital
# query, e.g. file records.
# On another hand, the most common use case where sort fails is
# getting file records, and I can add one compound key to ease sort
# but I can't add another compound key on array field, e.g. run
common_idx = [[('qhash', DESCENDING), ('file.name', DESCENDING)]]
create_indexes(self.merge, index_list + common_idx)
# thread which clean-up DAS collections
thname = 'mongocache_cleanup'
cols = [config['dasdb']['cachecollection'],
config['dasdb']['mrcollection'],
config['dasdb']['mergecollection']]
def __init__(self, config):
self.config = config
self.emptyset_expire = \
expire_timestamp(config['das'].get('emptyset_expire', 5))
self.dburi = config['mongodb']['dburi']
self.cache_size = config['mongodb']['bulkupdate_size']
self.dbname = config['dasdb']['dbname']
self.verbose = config['verbose']
self.logger = PrintManager('DASMongocache', self.verbose)
self.mapping = config['dasmapping']
self.logging = config['dasdb'].get('logging', False)
self.rec_ttl = config['dasdb'].get('record_ttl', 24 * 60 * 60)
self.del_ttl = config['dasdb'].get('delta_ttl', 60)
self.cleanup_del_ttl = config['dasdb'].get('cleanup_delta_ttl', 3600)
self.retry = config['dasdb'].get('retry', 3)
self.das_son_manipulator = DAS_SONManipulator()
# Initialize MongoDB connection
self.col_ = self.config['dasdb']['cachecollection']
self.mrcol_ = self.config['dasdb']['mrcollection']
self.merge_ = self.config['dasdb']['mergecollection']
self.gfs = db_gridfs(self.dburi)
msg = "%s@%s" % (self.dburi, self.dbname)
self.logger.info(msg)
# ensure that we have the following indexes
common_idx = [
('file.name', DESCENDING),
('dataset.name', DESCENDING),
('block.name', DESCENDING),
('run.run_number', DESCENDING),
]
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('das.system', ASCENDING), ('qhash', DESCENDING),
('das.record', ASCENDING)]
create_indexes(self.col, index_list + common_idx)
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('qhash', DESCENDING), ('das.record', ASCENDING),
('das.ts', ASCENDING)]
create_indexes(self.merge, index_list)
# NOTE: I found that creating index in merge collection leads to
# MongoDB error when records contains multiple arrays on indexed
# keys. For example, when we query file,run,lumi both file and run
# are arrays in MongoDB. In this case the final sort in MongoDB
# bark with the following message:
# cannot sort with keys that are parallel arrays
# it looks like that there is no fix for that yet
# see
# http://stackoverflow.com/questions/6516725/how-do-i-index-two-arrays-in-mongodb
# therefore I temporary disabled create_indexes call on merge
# collection which was used to have index to ease final sort,
# especially in a case when a lot of records correspond to inital
# query, e.g. file records.
# On another hand, the most common use case where sort fails is
# getting file records, and I can add one compound key to ease sort
# but I can't add another compound key on array field, e.g. run
common_idx = [[('qhash', DESCENDING), ('file.name', DESCENDING)]]
create_indexes(self.merge, index_list + common_idx)
# thread which clean-up DAS collections
thname = 'mongocache_cleanup'
cols = [
config['dasdb']['cachecollection'],
config['dasdb']['mrcollection'], config['dasdb']['mergecollection']
]