def __init__(self, config=None, logger=None): """ Initialize MSManager class with given configuration, logger, ReqMgr2/ReqMgrAux/PhEDEx/Rucio objects, and start transferor and monitoring threads. :param config: reqmgr2ms service configuration :param logger: """ self.config = config self.logger = getMSLogger(getattr(config, 'verbose', False), logger) self._parseConfig(config) self.logger.info("Configuration including default values:\n%s", self.msConfig) self.statusTrans = {} self.statusMon = {} # initialize transferor module if 'transferor' in self.services: self.msTransferor = MSTransferor(self.msConfig, logger=self.logger) thname = 'MSTransferor' self.transfThread = start_new_thread( thname, daemon, (self.transferor, 'assigned', self.msConfig['interval'], self.logger)) self.logger.debug("### Running %s thread %s", thname, self.transfThread.running()) # initialize monitoring module if 'monitor' in self.services: self.msMonitor = MSMonitor(self.msConfig, logger=self.logger) thname = 'MSMonitor' self.monitThread = start_new_thread( thname, daemon, (self.monitor, 'staging', self.msConfig['interval'], self.logger)) self.logger.debug("+++ Running %s thread %s", thname, self.monitThread.running())
def __init__(self, config=None, logger=None): """ Setup a bunch of things, like: * logger for this service * initialize all the necessary service helpers * fetch the unified configuration from central couch * update the unified configuration with some deployment and default settings * start both transfer and monitor threads :param config: reqmgr2ms service configuration :param logger: """ self.uConfig = {} self.config = config self.logger = getMSLogger(getattr(config, 'verbose', False), logger) self._parseConfig(config) self.logger.info("Configuration including default values:\n%s", self.msConfig) self.reqmgr2 = ReqMgr(self.msConfig['reqmgrUrl'], logger=self.logger) self.reqmgrAux = ReqMgrAux(self.msConfig['reqmgrUrl'], httpDict={'cacheduration': 60}, logger=self.logger) # transferor has to look at workflows in assigned status self.msTransferor = MSTransferor(self.msConfig, "assigned", logger=self.logger) ### Last but not least, get the threads started thname = 'MSTransferor' self.transfThread = start_new_thread( thname, daemon, (self.transferor, 'assigned', self.msConfig['interval'], self.logger)) self.logger.debug("### Running %s thread %s", thname, self.transfThread.running()) thname = 'MSTransferorMonit' self.monitThread = start_new_thread( thname, daemon, (self.monitor, 'staging', self.msConfig['interval'] * 2, self.logger)) self.logger.debug("+++ Running %s thread %s", thname, self.monitThread.running())
def __init__(self, svc, group='DataOps', readOnly=True, interval=60, logger=None): if logger: self.logger = logger else: self.logger = logging.getLogger('reqmgr2ms:MSManager') self.logger.setLevel(logging.DEBUG) logging.basicConfig() self.phedex = PhEDEx() # eventually will change to Rucio self.group = group self.readOnly = readOnly self.svc = svc # Services: ReqMgr, ReqMgrAux thname = 'MSTransferor' self.thr = start_new_thread(thname, daemon, (self.transferor, 'assigned', interval, self.logger)) self.logger.debug("### Running %s thread %s", thname, self.thr.running()) thname = 'MSTransferorMonit' self.ms_monit = start_new_thread(thname, daemon, (self.monit, 'staging', interval, self.logger)) self.logger.debug("+++ Running %s thread %s", thname, self.ms_monit.running()) self.logger.info("MSManager, group=%s, interval=%s", group, interval)
def __init__(self, group='DataOps', dbFileName=None, interval=10, verbose=False): self.verbose = verbose if not dbFileName: fobj = tempfile.NamedTemporaryFile() dbFileName = '%s.db' % fobj.name self.store = RequestStore(dbFileName) self.phedex = PhEDEx() self.group = group thname = 'RequestManager monitor' self.thr = start_new_thread(thname, checkRequests, \ (self.checkStatus, self.store, interval)) if verbose: print("### Running %s thread, running: %s" % (thname, self.thr.running())) print("RequestManager, group=%s, db=%s, interval=%s" % (group, dbFileName, interval))
def __init__(self, config=None, logger=None): """ Initialize MSManager class with given configuration, logger, ReqMgr2/ReqMgrAux/PhEDEx/Rucio objects, and start transferor and monitoring threads. :param config: reqmgr2ms service configuration :param logger: """ self.config = config self.logger = getMSLogger(getattr(config, 'verbose', False), logger) self._parseConfig(config) self.logger.info("Configuration including default values:\n%s", self.msConfig) self.statusTrans = {} self.statusMon = {} self.statusOutput = {} self.statusRuleCleaner = {} # initialize transferor module if 'transferor' in self.services: self.msTransferor = MSTransferor(self.msConfig, logger=self.logger) thname = 'MSTransferor' self.transfThread = start_new_thread( thname, daemon, (self.transferor, 'assigned', self.msConfig['interval'], self.logger)) self.logger.info("### Running %s thread %s", thname, self.transfThread.running()) # initialize monitoring module if 'monitor' in self.services: self.msMonitor = MSMonitor(self.msConfig, logger=self.logger) thname = 'MSMonitor' self.monitThread = start_new_thread( thname, daemon, (self.monitor, 'staging', self.msConfig['interval'], self.logger)) self.logger.info("+++ Running %s thread %s", thname, self.monitThread.running()) # initialize output module if 'output' in self.services: reqStatus = ['closed-out', 'announced'] # thread safe cache to keep the last X requests processed in MSOutput requestNamesCached = deque( maxlen=self.msConfig.get("cacheRequestSize", 10000)) thname = 'MSOutputConsumer' self.msOutputConsumer = MSOutput(self.msConfig, mode=thname, reqCache=requestNamesCached, logger=self.logger) # set the consumer to run twice faster than the producer consumerInterval = self.msConfig['interval'] // 2 self.outputConsumerThread = start_new_thread( thname, daemon, (self.outputConsumer, reqStatus, consumerInterval, self.logger)) self.logger.info("=== Running %s thread %s", thname, self.outputConsumerThread.running()) thname = 'MSOutputProducer' self.msOutputProducer = MSOutput(self.msConfig, mode=thname, reqCache=requestNamesCached, logger=self.logger) self.outputProducerThread = start_new_thread( thname, daemon, (self.outputProducer, reqStatus, self.msConfig['interval'], self.logger)) self.logger.info("=== Running %s thread %s", thname, self.outputProducerThread.running()) # initialize rule cleaner module if 'ruleCleaner' in self.services: reqStatus = ['announced', 'aborted-completed', 'rejected'] self.msRuleCleaner = MSRuleCleaner(self.msConfig, logger=self.logger) thname = 'MSRuleCleaner' self.ruleCleanerThread = start_new_thread( thname, daemon, (self.ruleCleaner, reqStatus, self.msConfig['interval'], self.logger)) self.logger.info("--- Running %s thread %s", thname, self.ruleCleanerThread.running())