def dbs_daemon(self, config):
"""Start DBS daemon if it is requested via DAS configuration"""
try:
main_dbs_url = self.dbs_url
dbs_urls = []
print "### DBS URL:", self.dbs_url
print "### DBS instances:", self.dbs_instances
if not self.dbs_url or not self.dbs_instances:
return # just quit
for inst in self.dbs_instances:
dbs_urls.append((main_dbs_url.replace(self.dbs_global, inst), inst))
interval = config.get("dbs_daemon_interval", 3600)
dbsexpire = config.get("dbs_daemon_expire", 3600)
preserve_dbs_col = config.get("preserve_on_restart", False)
dbs_config = {"expire": dbsexpire, "preserve_on_restart": preserve_dbs_col}
if self.dataset_daemon:
for dbs_url, inst in dbs_urls:
dbsmgr = DBSDaemon(dbs_url, self.dburi, dbs_config)
self.dbsmgr[(dbs_url, inst)] = dbsmgr
def dbs_updater(_dbsmgr, interval):
"""DBS updater daemon"""
while True:
try:
_dbsmgr.update()
except:
pass
time.sleep(interval)
print "### Start DBSDaemon for %s" % dbs_url
thname = "dbs_updater:%s" % dbs_url
start_new_thread(thname, dbs_updater, (dbsmgr, interval))
except Exception as exc:
print_exc(exc)
def __init__(self, dasconfig):
DASWebManager.__init__(self, dasconfig)
config = dasconfig['web_server']
self.pid_pat = re.compile(r'^[a-z0-9]{32}')
# TODO: self.base shall be automatically included in all tmpls
self.base = config['url_base']
self.interval = config.get('status_update', 2500)
self.engine = config.get('engine', None)
self.check_clients = config.get('check_clients', False)
nworkers = config['web_workers']
self.hot_thr = config.get('hot_threshold', 3000)
self.dasconfig = dasconfig
self.dburi = self.dasconfig['mongodb']['dburi']
self.lifetime = self.dasconfig['mongodb']['lifetime']
self.queue_limit = config.get('queue_limit', 50)
qtype = config.get('qtype', 'Queue')
qfreq = config.get('qfreq', 5)
if qtype not in ['Queue', 'PriorityQueue']:
msg = 'Wrong queue type, qtype=%s' % qtype
raise Exception(msg)
# if self.engine:
# thr_name = 'DASWebService:PluginTaskManager'
# self.taskmgr = PluginTaskManager(bus=self.engine, \
# nworkers=nworkers, name=thr_name, qtype=qtype, \
# qfreq=qfreq)
# self.taskmgr.subscribe()
# else:
# thr_name = 'DASWebService:TaskManager'
# self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, \
# qtype=qtype, qfreq=qfreq)
thr_name = 'DASWebService:TaskManager'
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, \
qtype=qtype, qfreq=qfreq)
self.adjust = config.get('adjust_input', False)
self.dasmgr = None # defined at run-time via self.init()
self.reqmgr = None # defined at run-time via self.init()
self.daskeys = [] # defined at run-time via self.init()
self.colors = {} # defined at run-time via self.init()
self.dbs_url = None # defined at run-time via self.init()
self.dbs_global = None # defined at run-time via self.init()
self.dbs_instances = [] # defined at run-time via self.init()
self.kws = None # defined at run-time via self.init()
self.q_rewriter = None # defined at run-time via self.init()
self.dataset_daemon = None
self.dbsmgr = {} # dbs_urls vs dbs_daemons, defined at run-time
self.daskeyslist = [] # list of DAS keys
self.init()
self.dbs_init(config)
# Monitoring thread which performs auto-reconnection
thname = 'dascore_monitor'
start_new_thread(thname, dascore_monitor, \
({'das':self.dasmgr, 'uri':self.dburi}, self.init, 5))
def __init__(self, dasconfig):
DASWebManager.__init__(self, dasconfig)
self.dasconfig = dasconfig
self.dburi = self.dasconfig['mongodb']['dburi']
self.dasmgr = None # defined at run-time via self.init()
self.kws = None # defined at run-time via self.init()
self.init()
# Monitoring thread which performs auto-reconnection
thname = 'dascore_monitor_kws'
start_new_thread(thname, dascore_monitor, ({'das': self.dasmgr,
'uri': self.dburi},
self.init, 5))
def __init__(self, dasconfig):
DASWebManager.__init__(self, dasconfig)
self.dasconfig = dasconfig
self.dburi = self.dasconfig['mongodb']['dburi']
self.dasmgr = None # defined at run-time via self.init()
self.kws = None # defined at run-time via self.init()
self.init()
# Monitoring thread which performs auto-reconnection
thname = 'dascore_monitor_kws'
start_new_thread(thname, dascore_monitor, ({
'das': self.dasmgr,
'uri': self.dburi
}, self.init, 5))
def init(self):
"""Takes care of MongoDB connection"""
try:
indexes = [('dataset', DESCENDING), ('site', DESCENDING),
('ts', DESCENDING)]
for index in indexes:
create_indexes(self.coll, [index])
dasmapping = DASMapping(self.dasconfig)
service_name = self.config.get('name', 'combined')
service_api = self.config.get('api', 'dataset4site_release')
mapping = dasmapping.servicemap(service_name)
self.urls = mapping[service_api]['services']
self.expire = mapping[service_api]['expire']
services = self.dasconfig['services']
if not self.wthr:
# Worker thread which update dbs/phedex DB
self.wthr = start_new_thread('dbs_phedex_worker', worker, \
(self.urls, which_dbs, self.uri, \
self.dbname, self.collname, self.expire))
msg = "### DBSPhedexService:init started"
print(msg)
except Exception as exc:
print("### Fail DBSPhedexService:init\n", str(exc))
self.urls = None
self.expire = 60
self.wthr = None
def __init__(self, dasconfig):
DASWebManager.__init__(self, dasconfig)
config = dasconfig["web_server"]
self.pid_pat = re.compile(r"^[a-z0-9]{32}")
self.base = config["url_base"]
self.interval = config.get("status_update", 2500)
self.engine = config.get("engine", None)
self.check_clients = config.get("check_clients", False)
nworkers = config["web_workers"]
self.hot_thr = config.get("hot_threshold", 3000)
self.dasconfig = dasconfig
self.dburi = self.dasconfig["mongodb"]["dburi"]
self.lifetime = self.dasconfig["mongodb"]["lifetime"]
self.queue_limit = config.get("queue_limit", 50)
qtype = config.get("qtype", "Queue")
if qtype not in ["Queue", "PriorityQueue"]:
msg = "Wrong queue type, qtype=%s" % qtype
raise Exception(msg)
if self.engine:
thr_name = "DASWebService:PluginTaskManager"
self.taskmgr = PluginTaskManager(bus=self.engine, nworkers=nworkers, name=thr_name, qtype=qtype)
self.taskmgr.subscribe()
else:
thr_name = "DASWebService:TaskManager"
self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, qtype=qtype)
self.adjust = config.get("adjust_input", False)
self.dasmgr = None # defined at run-time via self.init()
self.reqmgr = None # defined at run-time via self.init()
self.daskeys = [] # defined at run-time via self.init()
self.colors = {} # defined at run-time via self.init()
self.dbs_url = None # defined at run-time via self.init()
self.dbs_global = None # defined at run-time via self.init()
self.kws = None # defined at run-time via self.init()
self.q_rewriter = None # defined at run-time via self.init()
self.dataset_daemon = config.get("dbs_daemon", False)
self.dbsmgr = {} # dbs_urls vs dbs_daemons, defined at run-time
self.daskeyslist = [] # list of DAS keys
self.init()
# Monitoring thread which performs auto-reconnection
thname = "dascore_monitor"
start_new_thread(thname, dascore_monitor, ({"das": self.dasmgr, "uri": self.dburi}, self.init, 5))
def __init__(self, config):
self.verbose = config["verbose"]
self.logger = PrintManager("DASMapping", self.verbose)
self.services = config["services"]
self.dburi = config["mongodb"]["dburi"]
self.dbname = config["mappingdb"]["dbname"]
self.colname = config["mappingdb"]["collname"]
self.map_test = config.get("map_test", True)
self.main_dbs = config["das"].get("main_dbs", "dbs")
self.dbsinsts = config["das"].get("dbs_instances", [])
msg = "%s@%s" % (self.dburi, self.dbname)
self.logger.info(msg)
self.init()
self.on_reload = Event()
# Monitoring thread which performs auto-reconnection to MongoDB
thname = "mappingdb_monitor"
sleep = 5
reload_time = config["mappingdb"].get("reload_time", 86400)
reload_time_bad_maps = config["mappingdb"].get("reload_time_bad_maps", 120)
start_new_thread(
thname,
db_monitor,
(self.dburi, self.init, sleep, self.load_maps, reload_time, self.check_maps, reload_time_bad_maps),
)
self.daskeyscache = {} # to be filled at run time
self.systems = [] # to be filled at run time
self.dasmapscache = {} # to be filled at run time
self.keymap = {} # to be filled at run time
self.presentationcache = {} # to be filled at run time
self.reverse_presentation = {} # to be filled at run time
self.notationcache = {} # to be filled at run time
self.diffkeycache = {} # to be filled at run time
self.apicache = {} # to be filled at run time
self.dbs_global_url = None # to be determined at run time
self.dbs_inst_names = None # to be determined at run time
self.load_maps(notify=False)
def __init__(self, config):
self.config = config
self.emptyset_expire = \
expire_timestamp(config['das'].get('emptyset_expire', 5))
self.dburi = config['mongodb']['dburi']
self.cache_size = config['mongodb']['bulkupdate_size']
self.dbname = config['dasdb']['dbname']
self.verbose = config['verbose']
self.logger = PrintManager('DASMongocache', self.verbose)
self.mapping = config['dasmapping']
self.logging = config['dasdb'].get('logging', False)
self.rec_ttl = config['dasdb'].get('record_ttl', 24*60*60)
self.del_ttl = config['dasdb'].get('delta_ttl', 100)
self.retry = config['dasdb'].get('retry', 3)
self.das_son_manipulator = DAS_SONManipulator()
# Initialize MongoDB connection
self.col_ = self.config['dasdb']['cachecollection']
self.mrcol_ = self.config['dasdb']['mrcollection']
self.merge_ = self.config['dasdb']['mergecollection']
self.gfs = db_gridfs(self.dburi)
msg = "%s@%s" % (self.dburi, self.dbname)
self.logger.info(msg)
# ensure that we have the following indexes
common_idx = [
('file.name', DESCENDING),
('dataset.name', DESCENDING),
('block.name', DESCENDING),
('run.run_number', DESCENDING),
]
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('das.system', ASCENDING),
('qhash', DESCENDING),
('das.record', ASCENDING)]
create_indexes(self.col, index_list + common_idx)
index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
('qhash', DESCENDING),
('das.record', ASCENDING),
('das.ts', ASCENDING)]
create_indexes(self.merge, index_list)
# NOTE: I found that creating index in merge collection leads to
# MongoDB error when records contains multiple arrays on indexed
# keys. For example, when we query file,run,lumi both file and run
# are arrays in MongoDB. In this case the final sort in MongoDB
# bark with the following message:
# cannot sort with keys that are parallel arrays
# it looks like that there is no fix for that yet
# see
# http://stackoverflow.com/questions/6516725/how-do-i-index-two-arrays-in-mongodb
# therefore I temporary disabled create_indexes call on merge
# collection which was used to have index to ease final sort,
# especially in a case when a lot of records correspond to inital
# query, e.g. file records.
# On another hand, the most common use case where sort fails is
# getting file records, and I can add one compound key to ease sort
# but I can't add another compound key on array field, e.g. run
common_idx = [[('qhash', DESCENDING), ('file.name', DESCENDING)]]
create_indexes(self.merge, index_list + common_idx)
# thread which clean-up DAS collections
thname = 'mongocache_cleanup'
cols = [config['dasdb']['cachecollection'],
config['dasdb']['mrcollection'],
config['dasdb']['mergecollection']]
sleep = config['dasdb'].get('cleanup_interval', 600)
if config['dasdb'].get('cleanup_worker', True):
args = (self.dburi, self.dbname, cols, sleep)
start_new_thread(thname, cleanup_worker, args, unique=True)
