def dbs_daemon(self, config): """Start DBS daemon if it is requested via DAS configuration""" try: main_dbs_url = self.dbs_url dbs_urls = [] print "### DBS URL:", self.dbs_url print "### DBS instances:", self.dbs_instances if not self.dbs_url or not self.dbs_instances: return # just quit for inst in self.dbs_instances: dbs_urls.append((main_dbs_url.replace(self.dbs_global, inst), inst)) interval = config.get("dbs_daemon_interval", 3600) dbsexpire = config.get("dbs_daemon_expire", 3600) preserve_dbs_col = config.get("preserve_on_restart", False) dbs_config = {"expire": dbsexpire, "preserve_on_restart": preserve_dbs_col} if self.dataset_daemon: for dbs_url, inst in dbs_urls: dbsmgr = DBSDaemon(dbs_url, self.dburi, dbs_config) self.dbsmgr[(dbs_url, inst)] = dbsmgr def dbs_updater(_dbsmgr, interval): """DBS updater daemon""" while True: try: _dbsmgr.update() except: pass time.sleep(interval) print "### Start DBSDaemon for %s" % dbs_url thname = "dbs_updater:%s" % dbs_url start_new_thread(thname, dbs_updater, (dbsmgr, interval)) except Exception as exc: print_exc(exc)
def __init__(self, dasconfig): DASWebManager.__init__(self, dasconfig) config = dasconfig['web_server'] self.pid_pat = re.compile(r'^[a-z0-9]{32}') # TODO: self.base shall be automatically included in all tmpls self.base = config['url_base'] self.interval = config.get('status_update', 2500) self.engine = config.get('engine', None) self.check_clients = config.get('check_clients', False) nworkers = config['web_workers'] self.hot_thr = config.get('hot_threshold', 3000) self.dasconfig = dasconfig self.dburi = self.dasconfig['mongodb']['dburi'] self.lifetime = self.dasconfig['mongodb']['lifetime'] self.queue_limit = config.get('queue_limit', 50) qtype = config.get('qtype', 'Queue') qfreq = config.get('qfreq', 5) if qtype not in ['Queue', 'PriorityQueue']: msg = 'Wrong queue type, qtype=%s' % qtype raise Exception(msg) # if self.engine: # thr_name = 'DASWebService:PluginTaskManager' # self.taskmgr = PluginTaskManager(bus=self.engine, \ # nworkers=nworkers, name=thr_name, qtype=qtype, \ # qfreq=qfreq) # self.taskmgr.subscribe() # else: # thr_name = 'DASWebService:TaskManager' # self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, \ # qtype=qtype, qfreq=qfreq) thr_name = 'DASWebService:TaskManager' self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, \ qtype=qtype, qfreq=qfreq) self.adjust = config.get('adjust_input', False) self.dasmgr = None # defined at run-time via self.init() self.reqmgr = None # defined at run-time via self.init() self.daskeys = [] # defined at run-time via self.init() self.colors = {} # defined at run-time via self.init() self.dbs_url = None # defined at run-time via self.init() self.dbs_global = None # defined at run-time via self.init() self.dbs_instances = [] # defined at run-time via self.init() self.kws = None # defined at run-time via self.init() self.q_rewriter = None # defined at run-time via self.init() self.dataset_daemon = None self.dbsmgr = {} # dbs_urls vs dbs_daemons, defined at run-time self.daskeyslist = [] # list of DAS keys self.init() self.dbs_init(config) # Monitoring thread which performs auto-reconnection thname = 'dascore_monitor' start_new_thread(thname, dascore_monitor, \ ({'das':self.dasmgr, 'uri':self.dburi}, self.init, 5))
def __init__(self, dasconfig): DASWebManager.__init__(self, dasconfig) self.dasconfig = dasconfig self.dburi = self.dasconfig['mongodb']['dburi'] self.dasmgr = None # defined at run-time via self.init() self.kws = None # defined at run-time via self.init() self.init() # Monitoring thread which performs auto-reconnection thname = 'dascore_monitor_kws' start_new_thread(thname, dascore_monitor, ({'das': self.dasmgr, 'uri': self.dburi}, self.init, 5))
def __init__(self, dasconfig): DASWebManager.__init__(self, dasconfig) self.dasconfig = dasconfig self.dburi = self.dasconfig['mongodb']['dburi'] self.dasmgr = None # defined at run-time via self.init() self.kws = None # defined at run-time via self.init() self.init() # Monitoring thread which performs auto-reconnection thname = 'dascore_monitor_kws' start_new_thread(thname, dascore_monitor, ({ 'das': self.dasmgr, 'uri': self.dburi }, self.init, 5))
def init(self): """Takes care of MongoDB connection""" try: indexes = [('dataset', DESCENDING), ('site', DESCENDING), ('ts', DESCENDING)] for index in indexes: create_indexes(self.coll, [index]) dasmapping = DASMapping(self.dasconfig) service_name = self.config.get('name', 'combined') service_api = self.config.get('api', 'dataset4site_release') mapping = dasmapping.servicemap(service_name) self.urls = mapping[service_api]['services'] self.expire = mapping[service_api]['expire'] services = self.dasconfig['services'] if not self.wthr: # Worker thread which update dbs/phedex DB self.wthr = start_new_thread('dbs_phedex_worker', worker, \ (self.urls, which_dbs, self.uri, \ self.dbname, self.collname, self.expire)) msg = "### DBSPhedexService:init started" print(msg) except Exception as exc: print("### Fail DBSPhedexService:init\n", str(exc)) self.urls = None self.expire = 60 self.wthr = None
def __init__(self, dasconfig): DASWebManager.__init__(self, dasconfig) config = dasconfig["web_server"] self.pid_pat = re.compile(r"^[a-z0-9]{32}") self.base = config["url_base"] self.interval = config.get("status_update", 2500) self.engine = config.get("engine", None) self.check_clients = config.get("check_clients", False) nworkers = config["web_workers"] self.hot_thr = config.get("hot_threshold", 3000) self.dasconfig = dasconfig self.dburi = self.dasconfig["mongodb"]["dburi"] self.lifetime = self.dasconfig["mongodb"]["lifetime"] self.queue_limit = config.get("queue_limit", 50) qtype = config.get("qtype", "Queue") if qtype not in ["Queue", "PriorityQueue"]: msg = "Wrong queue type, qtype=%s" % qtype raise Exception(msg) if self.engine: thr_name = "DASWebService:PluginTaskManager" self.taskmgr = PluginTaskManager(bus=self.engine, nworkers=nworkers, name=thr_name, qtype=qtype) self.taskmgr.subscribe() else: thr_name = "DASWebService:TaskManager" self.taskmgr = TaskManager(nworkers=nworkers, name=thr_name, qtype=qtype) self.adjust = config.get("adjust_input", False) self.dasmgr = None # defined at run-time via self.init() self.reqmgr = None # defined at run-time via self.init() self.daskeys = [] # defined at run-time via self.init() self.colors = {} # defined at run-time via self.init() self.dbs_url = None # defined at run-time via self.init() self.dbs_global = None # defined at run-time via self.init() self.kws = None # defined at run-time via self.init() self.q_rewriter = None # defined at run-time via self.init() self.dataset_daemon = config.get("dbs_daemon", False) self.dbsmgr = {} # dbs_urls vs dbs_daemons, defined at run-time self.daskeyslist = [] # list of DAS keys self.init() # Monitoring thread which performs auto-reconnection thname = "dascore_monitor" start_new_thread(thname, dascore_monitor, ({"das": self.dasmgr, "uri": self.dburi}, self.init, 5))
def __init__(self, config): self.verbose = config["verbose"] self.logger = PrintManager("DASMapping", self.verbose) self.services = config["services"] self.dburi = config["mongodb"]["dburi"] self.dbname = config["mappingdb"]["dbname"] self.colname = config["mappingdb"]["collname"] self.map_test = config.get("map_test", True) self.main_dbs = config["das"].get("main_dbs", "dbs") self.dbsinsts = config["das"].get("dbs_instances", []) msg = "%s@%s" % (self.dburi, self.dbname) self.logger.info(msg) self.init() self.on_reload = Event() # Monitoring thread which performs auto-reconnection to MongoDB thname = "mappingdb_monitor" sleep = 5 reload_time = config["mappingdb"].get("reload_time", 86400) reload_time_bad_maps = config["mappingdb"].get("reload_time_bad_maps", 120) start_new_thread( thname, db_monitor, (self.dburi, self.init, sleep, self.load_maps, reload_time, self.check_maps, reload_time_bad_maps), ) self.daskeyscache = {} # to be filled at run time self.systems = [] # to be filled at run time self.dasmapscache = {} # to be filled at run time self.keymap = {} # to be filled at run time self.presentationcache = {} # to be filled at run time self.reverse_presentation = {} # to be filled at run time self.notationcache = {} # to be filled at run time self.diffkeycache = {} # to be filled at run time self.apicache = {} # to be filled at run time self.dbs_global_url = None # to be determined at run time self.dbs_inst_names = None # to be determined at run time self.load_maps(notify=False)
def __init__(self, config): self.config = config self.emptyset_expire = \ expire_timestamp(config['das'].get('emptyset_expire', 5)) self.dburi = config['mongodb']['dburi'] self.cache_size = config['mongodb']['bulkupdate_size'] self.dbname = config['dasdb']['dbname'] self.verbose = config['verbose'] self.logger = PrintManager('DASMongocache', self.verbose) self.mapping = config['dasmapping'] self.logging = config['dasdb'].get('logging', False) self.rec_ttl = config['dasdb'].get('record_ttl', 24*60*60) self.del_ttl = config['dasdb'].get('delta_ttl', 100) self.retry = config['dasdb'].get('retry', 3) self.das_son_manipulator = DAS_SONManipulator() # Initialize MongoDB connection self.col_ = self.config['dasdb']['cachecollection'] self.mrcol_ = self.config['dasdb']['mrcollection'] self.merge_ = self.config['dasdb']['mergecollection'] self.gfs = db_gridfs(self.dburi) msg = "%s@%s" % (self.dburi, self.dbname) self.logger.info(msg) # ensure that we have the following indexes common_idx = [ ('file.name', DESCENDING), ('dataset.name', DESCENDING), ('block.name', DESCENDING), ('run.run_number', DESCENDING), ] index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING), ('das.system', ASCENDING), ('qhash', DESCENDING), ('das.record', ASCENDING)] create_indexes(self.col, index_list + common_idx) index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING), ('qhash', DESCENDING), ('das.record', ASCENDING), ('das.ts', ASCENDING)] create_indexes(self.merge, index_list) # NOTE: I found that creating index in merge collection leads to # MongoDB error when records contains multiple arrays on indexed # keys. For example, when we query file,run,lumi both file and run # are arrays in MongoDB. In this case the final sort in MongoDB # bark with the following message: # cannot sort with keys that are parallel arrays # it looks like that there is no fix for that yet # see # http://stackoverflow.com/questions/6516725/how-do-i-index-two-arrays-in-mongodb # therefore I temporary disabled create_indexes call on merge # collection which was used to have index to ease final sort, # especially in a case when a lot of records correspond to inital # query, e.g. file records. # On another hand, the most common use case where sort fails is # getting file records, and I can add one compound key to ease sort # but I can't add another compound key on array field, e.g. run common_idx = [[('qhash', DESCENDING), ('file.name', DESCENDING)]] create_indexes(self.merge, index_list + common_idx) # thread which clean-up DAS collections thname = 'mongocache_cleanup' cols = [config['dasdb']['cachecollection'], config['dasdb']['mrcollection'], config['dasdb']['mergecollection']] sleep = config['dasdb'].get('cleanup_interval', 600) if config['dasdb'].get('cleanup_worker', True): args = (self.dburi, self.dbname, cols, sleep) start_new_thread(thname, cleanup_worker, args, unique=True)